From 7aab46c1817f82f8d280e2b3662c5671f13ec1cd Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 5 Feb 2025 15:22:26 +0000 Subject: [PATCH 01/80] Add exclude_options configuration for generation This adds a new plugin option `exclude_options` for use on generation. The excluded options are stripped from the Image creating a shallow clone of the FileDescriptors that make up the ImageFiles. Any imports that are no longer required due to options being removed are also removed. The option is applied for the plugin. Multiple filtered images that reference the original image may be created when exclude options are set. --- private/buf/bufgen/generator.go | 14 + private/bufpkg/bufconfig/buf_gen_yaml_file.go | 2 + .../bufconfig/buf_gen_yaml_file_test.go | 2 + .../bufconfig/generate_plugin_config.go | 36 + .../bufimage/bufimageutil/exclude_options.go | 749 ++++++++++++++++++ .../bufimageutil/exclude_options_test.go | 232 ++++++ private/bufpkg/bufimage/bufimageutil/tags.go | 9 + .../TestExcludeOptionImports/ExcludeBar.txtar | 485 ++++++++++++ .../TestExcludeOptionImports/ExcludeFoo.txtar | 485 ++++++++++++ .../ExcludeFooBar.txtar | 9 + .../NoneExcluded.txtar | 486 ++++++++++++ .../testdata/excludeoptionimports/a.proto | 12 + .../excludeoptionimports/options.proto | 14 + .../TestExcludeOptions/ExcludeAll.txtar | 546 +++++++++++++ .../TestExcludeOptions/ExcludeFoo.txtar | 557 +++++++++++++ .../TestExcludeOptions/ExcludeMessage.txtar | 573 ++++++++++++++ .../TestExcludeOptions/NoneExcluded.txtar | 575 ++++++++++++++ .../TestExcludeOptions/OnlyEnumValue.txtar | 553 +++++++++++++ .../TestExcludeOptions/OnlyFile.txtar | 549 +++++++++++++ .../TestExcludeOptions/OnlyOneOf.txtar | 549 +++++++++++++ .../testdata/excludeoptions/a.proto | 82 ++ .../testdata/excludeoptions/options.proto | 56 ++ 22 files changed, 6575 insertions(+) create mode 100644 private/bufpkg/bufimage/bufimageutil/exclude_options.go create mode 100644 private/bufpkg/bufimage/bufimageutil/exclude_options_test.go create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeBar.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFoo.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFooBar.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/NoneExcluded.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/a.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/options.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 34802ab484..7a2aa5dd92 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -26,6 +26,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" "github.com/bufbuild/buf/private/bufpkg/bufprotoplugin" "github.com/bufbuild/buf/private/bufpkg/bufprotoplugin/bufprotopluginos" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" @@ -235,6 +236,7 @@ func (g *generator) execPlugins( if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } + excludeOptions := currentPluginConfig.ExcludeOptions() response, err := g.execLocalPlugin( ctx, container, @@ -242,6 +244,7 @@ func (g *generator) execPlugins( currentPluginConfig, includeImports, includeWellKnownTypes, + excludeOptions, ) if err != nil { return err @@ -309,11 +312,21 @@ func (g *generator) execLocalPlugin( pluginConfig bufconfig.GeneratePluginConfig, includeImports bool, includeWellKnownTypes bool, + excludeOptions []string, ) (*pluginpb.CodeGeneratorResponse, error) { pluginImages, err := imageProvider.GetImages(Strategy(pluginConfig.Strategy())) if err != nil { return nil, err } + if len(excludeOptions) > 0 { + for i, pluginImage := range pluginImages { + pluginImage, err := bufimageutil.ExcludeOptions(pluginImage, excludeOptions...) + if err != nil { + return nil, err + } + pluginImages[i] = pluginImage + } + } requests, err := bufimage.ImagesToCodeGeneratorRequests( pluginImages, pluginConfig.Opt(), @@ -356,6 +369,7 @@ func (g *generator) execRemotePluginsV2( pluginConfigs []*remotePluginExecArgs, includeImportsOverride *bool, includeWellKnownTypesOverride *bool, + // TODO: add support for exclude_options. ) ([]*remotePluginExecutionResult, error) { requests := make([]*registryv1alpha1.PluginGenerationRequest, len(pluginConfigs)) for i, pluginConfig := range pluginConfigs { diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 1ce185fbef..737bcee0ba 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -514,6 +514,8 @@ type externalGeneratePluginConfigV2 struct { IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` // Strategy is only valid with ProtoBuiltin and Local. Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` + // ExcludeOptions removes options from the image. + ExcludeOptions []string `json:"exclude_options,omitempty" yaml:"exclude_options,omitempty"` } // externalGenerateManagedConfigV2 represents the managed mode config in a v2 buf.gen.yaml file. diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go b/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go index 0ac8117afd..4903120688 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go @@ -180,6 +180,8 @@ plugins: strategy: all include_imports: true include_wkt: true + exclude_options: + - buf.validate.oneof inputs: - git_repo: github.com/acme/weather branch: dev diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 3208cf0386..8fc996065d 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -112,6 +112,8 @@ type GeneratePluginConfig interface { // // This is not empty only when the plugin is remote. Revision() int + // ExcludeOptions returns the options to exclude. + ExcludeOptions() []string isGeneratePluginConfig() } @@ -123,6 +125,7 @@ func NewRemoteGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, revision int, ) (GeneratePluginConfig, error) { return newRemoteGeneratePluginConfig( @@ -131,6 +134,7 @@ func NewRemoteGeneratePluginConfig( opt, includeImports, includeWKT, + excludeOptions, revision, ) } @@ -142,6 +146,7 @@ func NewLocalOrProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, strategy *GenerateStrategy, ) (GeneratePluginConfig, error) { return newLocalOrProtocBuiltinGeneratePluginConfig( @@ -150,6 +155,7 @@ func NewLocalOrProtocBuiltinGeneratePluginConfig( opt, includeImports, includeWKT, + excludeOptions, strategy, ) } @@ -161,6 +167,7 @@ func NewLocalGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, strategy *GenerateStrategy, path []string, ) (GeneratePluginConfig, error) { @@ -170,6 +177,7 @@ func NewLocalGeneratePluginConfig( opt, includeImports, includeWKT, + excludeOptions, strategy, path, ) @@ -183,6 +191,7 @@ func NewProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, strategy *GenerateStrategy, protocPath []string, ) (GeneratePluginConfig, error) { @@ -192,6 +201,7 @@ func NewProtocBuiltinGeneratePluginConfig( opt, includeImports, includeWKT, + excludeOptions, strategy, protocPath, ) @@ -203,6 +213,7 @@ func NewGeneratePluginConfigWithIncludeImportsAndWKT( config GeneratePluginConfig, includeImports bool, includeWKT bool, + excludeOptions []string, ) (GeneratePluginConfig, error) { originalConfig, ok := config.(*generatePluginConfig) if !ok { @@ -215,6 +226,9 @@ func NewGeneratePluginConfigWithIncludeImportsAndWKT( if includeWKT { generatePluginConfig.includeWKT = true } + if len(excludeOptions) > 0 { + generatePluginConfig.excludeOptions = excludeOptions + } return &generatePluginConfig, nil } @@ -227,6 +241,7 @@ type generatePluginConfig struct { opts []string includeImports bool includeWKT bool + excludeOptions []string strategy *GenerateStrategy path []string protocPath []string @@ -258,6 +273,7 @@ func newGeneratePluginConfigFromExternalV1Beta1( opt, false, false, + nil, // TODO strategy, []string{externalConfig.Path}, ) @@ -268,6 +284,7 @@ func newGeneratePluginConfigFromExternalV1Beta1( opt, false, false, + nil, // TODO strategy, ) } @@ -327,6 +344,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, + nil, // TODO externalConfig.Revision, ) } @@ -339,6 +357,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, + nil, // TODO strategy, path, ) @@ -350,6 +369,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, + nil, // TODO strategy, protocPath, ) @@ -362,6 +382,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, + nil, // TODO strategy, ) } @@ -418,6 +439,7 @@ func newGeneratePluginConfigFromExternalV2( opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + externalConfig.ExcludeOptions, revision, ) case externalConfig.Local != nil: @@ -438,6 +460,7 @@ func newGeneratePluginConfigFromExternalV2( opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + externalConfig.ExcludeOptions, parsedStrategy, path, ) @@ -455,6 +478,7 @@ func newGeneratePluginConfigFromExternalV2( opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + externalConfig.ExcludeOptions, parsedStrategy, protocPath, ) @@ -469,6 +493,7 @@ func newRemoteGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, revision int, ) (*generatePluginConfig, error) { if includeWKT && !includeImports { @@ -490,6 +515,7 @@ func newRemoteGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, + excludeOptions: excludeOptions, }, nil } @@ -499,6 +525,7 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, strategy *GenerateStrategy, ) (*generatePluginConfig, error) { if includeWKT && !includeImports { @@ -512,6 +539,7 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, + excludeOptions: excludeOptions, }, nil } @@ -521,6 +549,7 @@ func newLocalGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, strategy *GenerateStrategy, path []string, ) (*generatePluginConfig, error) { @@ -539,6 +568,7 @@ func newLocalGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, + excludeOptions: excludeOptions, }, nil } @@ -548,6 +578,7 @@ func newProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + excludeOptions []string, strategy *GenerateStrategy, protocPath []string, ) (*generatePluginConfig, error) { @@ -563,6 +594,7 @@ func newProtocBuiltinGeneratePluginConfig( strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, + excludeOptions: excludeOptions, }, nil } @@ -590,6 +622,10 @@ func (p *generatePluginConfig) IncludeWKT() bool { return p.includeWKT } +func (p *generatePluginConfig) ExcludeOptions() []string { + return p.excludeOptions +} + func (p *generatePluginConfig) Strategy() GenerateStrategy { if p.strategy == nil { return GenerateStrategyDirectory diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options.go b/private/bufpkg/bufimage/bufimageutil/exclude_options.go new file mode 100644 index 0000000000..d60eb00185 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options.go @@ -0,0 +1,749 @@ +// Copyright 2020-2025 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimageutil + +import ( + "fmt" + "slices" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/protoencoding" + "github.com/bufbuild/protocompile/walk" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" +) + +// ExcludeOptions returns an Image that omits all provided options. +// +// If the Image has none of the provided options, the original Image is returned. +// If the Image has any of the provided options, a new Image is returned. +// If an import is no longer imported by any file, it is removed from the Image. +// +// The returned Image will be a shallow copy of the original Image. +// The FileDescriptors in the returned Image.Files will be a shallow copy of the original FileDescriptorProto. +// +// The options are specified by their type name, e.g. "buf.validate.message". +func ExcludeOptions(image bufimage.Image, options ...string) (bufimage.Image, error) { + if len(options) == 0 { + return image, nil + } + imageFiles := image.Files() + newImageFiles := make([]bufimage.ImageFile, 0, len(image.Files())) + importsByFilePath := make(map[string]struct{}) + + optionsByName := make(map[string]struct{}, len(options)) + for _, option := range options { + optionsByName[option] = struct{}{} + } + optionsFilter := func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool { + _, ok := optionsByName[string(fd.FullName())] + return !ok + } + filter := newOptionsFilter(image.Resolver(), optionsFilter) + + // Loop over image files in revserse DAG order. Imports that are no longer + // imported by a previous file are dropped from the image. + dirty := false + for i := len(image.Files()) - 1; i >= 0; i-- { + imageFile := imageFiles[i] + imageFilePath := imageFile.Path() + if imageFile.IsImport() { + // Check if this import is still used. + if _, isImportUsed := importsByFilePath[imageFilePath]; !isImportUsed { + continue + } + } + oldFileDescriptor := imageFile.FileDescriptorProto() + newFileDescriptor, err := filter.fileDescriptor(oldFileDescriptor, imageFilePath) + if err != nil { + return nil, err + } + for _, filePath := range newFileDescriptor.Dependency { + importsByFilePath[filePath] = struct{}{} + } + // Create a new image file with the modified file descriptor. + if newFileDescriptor != oldFileDescriptor { + dirty = true + imageFile, err = bufimage.NewImageFile( + newFileDescriptor, + imageFile.FullName(), + imageFile.CommitID(), + imageFile.ExternalPath(), + imageFile.LocalPath(), + imageFile.IsImport(), + imageFile.IsSyntaxUnspecified(), + imageFile.UnusedDependencyIndexes(), + ) + if err != nil { + return nil, err + } + } + newImageFiles = append(newImageFiles, imageFile) + } + if dirty { + // Revserse the image files back to DAG order. + slices.Reverse(newImageFiles) + return bufimage.NewImage(newImageFiles) + } + return image, nil +} + +// protoOptionsFilter is a filter for options. +// +// This filter is applied to FileDescriptorProto to remove options. A new shallow +// copy of the FileDescriptorProto is returned with the options removed. If no +// options are removed, the original FileDescriptorProto is returned. +type protoOptionsFilter struct { + // resolver is used to resvole parent files for required types. + resolver protodesc.Resolver + // optionsFilter returns true if the option should be kept. + optionsFilter func(protoreflect.FieldDescriptor, protoreflect.Value) bool + + // sourcePathsRemaps is a trie of source path remaps. Reset per file iteration. + sourcePathRemaps sourcePathsRemapTrie + // requiredTypes is a set of required types. Reset per file iteration. + requiredTypes map[protoreflect.FullName]struct{} +} + +func newOptionsFilter(resolver protoencoding.Resolver, optionsFilter func(protoreflect.FieldDescriptor, protoreflect.Value) bool) *protoOptionsFilter { + return &protoOptionsFilter{ + resolver: resolver, + optionsFilter: optionsFilter, + } +} + +func (f *protoOptionsFilter) fileDescriptor( + fileDescriptor *descriptorpb.FileDescriptorProto, + filePath string, +) (*descriptorpb.FileDescriptorProto, error) { + if f.requiredTypes == nil { + f.requiredTypes = make(map[protoreflect.FullName]struct{}) + } + // Check file options. + if options := fileDescriptor.GetOptions(); options != nil { + optionsPath := []int32{fileOptionsTag} + if err := f.options(options, optionsPath); err != nil { + return nil, err + } + } + // Walk the file descriptor, collecting required types and marking options for deletion. + if err := walk.DescriptorProtosWithPath(fileDescriptor, f.visit); err != nil { + return nil, err + } + if len(f.sourcePathRemaps) == 0 { + return fileDescriptor, nil // No changes required. + } + + // Recursively apply the source path remaps. + newFileDescriptor := shallowClone(fileDescriptor) + if err := remapFileDescriptor( + newFileDescriptor, + f.sourcePathRemaps, + ); err != nil { + return nil, err + } + // Convert the required types to imports. + fileImports := make(map[string]struct{}, len(fileDescriptor.Dependency)) + for requiredType := range f.requiredTypes { + descriptor, err := f.resolver.FindDescriptorByName(requiredType) + if err != nil { + return nil, fmt.Errorf("couldn't find type %s: %w", requiredType, err) + } + importPath := descriptor.ParentFile().Path() + if _, ok := fileImports[importPath]; ok || importPath == filePath { + continue + } + fileImports[importPath] = struct{}{} + } + // Fix imports. + if len(fileImports) != len(fileDescriptor.Dependency) { + i := 0 + dependencyPath := []int32{fileDependencyTag} + dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) + newFileDescriptor.Dependency = make([]string, 0, len(fileImports)) + for index, dependency := range fileDescriptor.Dependency { + path := append(dependencyPath, int32(index)) + if _, ok := fileImports[dependency]; ok { + newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, dependency) + dependencyChanges[index] = int32(i) + if i != index { + f.sourcePathRemaps.markMoved(path, int32(i)) + } + i++ + } else { + f.sourcePathRemaps.markDeleted(path) + dependencyChanges[index] = -1 + } + } + publicDependencyPath := []int32{filePublicDependencyTag} + newFileDescriptor.PublicDependency = make([]int32, 0, len(fileDescriptor.PublicDependency)) + for index, publicDependency := range fileDescriptor.PublicDependency { + path := append(publicDependencyPath, int32(index)) + newPublicDependency := dependencyChanges[publicDependency] + if newPublicDependency == -1 { + f.sourcePathRemaps.markDeleted(path) + } else { + newFileDescriptor.PublicDependency = append(newFileDescriptor.PublicDependency, newPublicDependency) + if newPublicDependency != int32(index) { + f.sourcePathRemaps.markMoved(path, newPublicDependency) + } + } + } + weakDependencyPath := []int32{fileWeakDependencyTag} + newFileDescriptor.WeakDependency = make([]int32, 0, len(fileDescriptor.WeakDependency)) + for index, weakDependency := range fileDescriptor.WeakDependency { + path := append(weakDependencyPath, int32(index)) + newWeakDependency := dependencyChanges[weakDependency] + if newWeakDependency == -1 { + f.sourcePathRemaps.markDeleted(path) + } else { + newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, newWeakDependency) + if newWeakDependency != int32(index) { + f.sourcePathRemaps.markMoved(path, newWeakDependency) + } + } + } + } + // Remap the source code info. + if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { + newLocations := make([]*descriptorpb.SourceCodeInfo_Location, 0, len(locations)) + for _, location := range locations { + oldPath := location.Path + newPath, noComment := f.sourcePathRemaps.newPath(oldPath) + if newPath == nil { + continue + } + if !slices.Equal(oldPath, newPath) || noComment { + location = shallowClone(location) + location.Path = newPath + } + if noComment { + location.LeadingDetachedComments = nil + location.LeadingComments = nil + location.TrailingComments = nil + } + newLocations = append(newLocations, location) + } + newFileDescriptor.SourceCodeInfo = &descriptorpb.SourceCodeInfo{ + Location: newLocations, + } + } + // Cleanup. + f.sourcePathRemaps = f.sourcePathRemaps[:0] + for key := range f.requiredTypes { + delete(f.requiredTypes, key) + } + return newFileDescriptor, nil +} + +func (f *protoOptionsFilter) visit(fullName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor proto.Message) error { + switch descriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + // File options are handled at the top level, before walking the file. + return nil + case *descriptorpb.DescriptorProto: + optionsPath := append(sourcePath, messageOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.FieldDescriptorProto: + // Add the field type to the required types. + if err := f.addFieldType(descriptor); err != nil { + return err + } + optionsPath := append(sourcePath, fieldOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.OneofDescriptorProto: + optionsPath := append(sourcePath, oneofOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.EnumDescriptorProto: + optionsPath := append(sourcePath, enumOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.EnumValueDescriptorProto: + optionsPath := append(sourcePath, enumValueOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.ServiceDescriptorProto: + optionsPath := append(sourcePath, serviceOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.MethodDescriptorProto: + optionsPath := append(sourcePath, methodOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + case *descriptorpb.DescriptorProto_ExtensionRange: + optionsPath := append(sourcePath, extensionRangeOptionsTag) + return f.options(descriptor.GetOptions(), optionsPath) + default: + return fmt.Errorf("unexpected message type %T", descriptor) + } +} + +func (f *protoOptionsFilter) options( + optionsMessage proto.Message, + optionsPath protoreflect.SourcePath, +) error { + if optionsMessage == nil { + return nil + } + options := optionsMessage.ProtoReflect() + if !options.IsValid() { + return nil // No options to strip. + } + + numFieldsToKeep := 0 + options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + if !f.optionsFilter(fd, v) { + // Remove this option. + optionPath := append(optionsPath, int32(fd.Number())) + f.sourcePathRemaps.markDeleted(optionPath) + return true + } + numFieldsToKeep++ + if fd.IsExtension() { + // Add the extension type to the required types. + f.requiredTypes[fd.FullName()] = struct{}{} + } + return true + }) + if numFieldsToKeep == 0 { + // No options to keep. + f.sourcePathRemaps.markDeleted(optionsPath) + } + return nil +} + +func (f *protoOptionsFilter) addFieldType(field *descriptorpb.FieldDescriptorProto) error { + if field.Extendee != nil { + // This is an extension field. + extendee := strings.TrimPrefix(field.GetExtendee(), ".") + f.requiredTypes[protoreflect.FullName(extendee)] = struct{}{} + } + switch field.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_ENUM, + descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, + descriptorpb.FieldDescriptorProto_TYPE_GROUP: + typeName := strings.TrimPrefix(field.GetTypeName(), ".") + f.requiredTypes[protoreflect.FullName(typeName)] = struct{}{} + case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, + descriptorpb.FieldDescriptorProto_TYPE_FLOAT, + descriptorpb.FieldDescriptorProto_TYPE_INT64, + descriptorpb.FieldDescriptorProto_TYPE_UINT64, + descriptorpb.FieldDescriptorProto_TYPE_INT32, + descriptorpb.FieldDescriptorProto_TYPE_FIXED64, + descriptorpb.FieldDescriptorProto_TYPE_FIXED32, + descriptorpb.FieldDescriptorProto_TYPE_BOOL, + descriptorpb.FieldDescriptorProto_TYPE_STRING, + descriptorpb.FieldDescriptorProto_TYPE_BYTES, + descriptorpb.FieldDescriptorProto_TYPE_UINT32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, + descriptorpb.FieldDescriptorProto_TYPE_SINT32, + descriptorpb.FieldDescriptorProto_TYPE_SINT64: + // nothing to follow, custom options handled above. + default: + return fmt.Errorf("unknown field type %d", field.GetType()) + } + return nil +} + +// TODO: rewrite using protoreflect. +func remapFileDescriptor( + fileDescriptor *descriptorpb.FileDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case fileDependencyTag: + // Dependencies are handled after message remaps. + return fmt.Errorf("unexpected dependency move %d to %d", remapNode.oldIndex, remapNode.newIndex) + case filePublicDependencyTag: + return fmt.Errorf("unexpected public dependency move %d to %d", remapNode.oldIndex, remapNode.newIndex) + case fileWeakDependencyTag: + return fmt.Errorf("unexpected weak dependency move %d to %d", remapNode.oldIndex, remapNode.newIndex) + case fileMessagesTag: + if err := remapSlice( + &fileDescriptor.MessageType, + remapNode, + remapMessageDescriptor, + ); err != nil { + return err + } + case fileEnumsTag: + if err := remapSlice( + &fileDescriptor.EnumType, + remapNode, + remapEnumDescriptor, + ); err != nil { + return err + } + case fileServicesTag: + if err := remapSlice( + &fileDescriptor.Service, + remapNode, + remapServiceDescriptor, + ); err != nil { + return err + } + case fileExtensionsTag: + if err := remapSlice( + &fileDescriptor.Extension, + remapNode, + remapFieldDescriptor, + ); err != nil { + return err + } + case fileOptionsTag: + if err := remapMessage( + &fileDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected file index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapMessageDescriptor( + messageDescriptor *descriptorpb.DescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case messageFieldsTag: + if err := remapSlice( + &messageDescriptor.Field, + remapNode, + remapFieldDescriptor, + ); err != nil { + return err + } + case messageNestedMessagesTag: + if err := remapSlice( + &messageDescriptor.NestedType, + remapNode, + remapMessageDescriptor, + ); err != nil { + return err + } + case messageEnumsTag: + if err := remapSlice( + &messageDescriptor.EnumType, + remapNode, + remapEnumDescriptor, + ); err != nil { + return err + } + case messageExtensionsTag: + if err := remapSlice( + &messageDescriptor.Extension, + remapNode, + remapFieldDescriptor, + ); err != nil { + return err + } + case messageOptionsTag: + if err := remapMessage( + &messageDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + case messageOneofsTag: + if err := remapSlice( + &messageDescriptor.OneofDecl, + remapNode, + remapOneofDescriptor, + ); err != nil { + return err + } + case messageExtensionRangesTag: + if err := remapSlice( + &messageDescriptor.ExtensionRange, + remapNode, + remapExtensionRangeDescriptor, + ); err != nil { + return err + } + case messageReservedRangesTag: + // TODO: handle reserved ranges tag. + return fmt.Errorf("unexpected reserved tags move %d to %d", remapNode.oldIndex, remapNode.newIndex) + case messageReservedNamesTag: + // TODO: handle reserved names. + return fmt.Errorf("unexpected reserved names move %d to %d", remapNode.oldIndex, remapNode.newIndex) + default: + return fmt.Errorf("unexpected message index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapFieldDescriptor( + fieldDescriptor *descriptorpb.FieldDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case fieldOptionsTag: + if err := remapMessage( + &fieldDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected field index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapOneofDescriptor( + oneofDescriptor *descriptorpb.OneofDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case oneofOptionsTag: + if err := remapMessage( + &oneofDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected oneof index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapEnumDescriptor( + enumDescriptor *descriptorpb.EnumDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case enumValuesTag: + if err := remapSlice( + &enumDescriptor.Value, + remapNode, + remapEnumValueDescriptor, + ); err != nil { + return err + } + case enumOptionsTag: + if err := remapMessage( + &enumDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected enum index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapEnumValueDescriptor( + enumValueDescriptor *descriptorpb.EnumValueDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case enumValueOptionsTag: + if err := remapMessage( + &enumValueDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected enum value index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapExtensionRangeDescriptor( + extensionRangeDescriptor *descriptorpb.DescriptorProto_ExtensionRange, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case extensionRangeOptionsTag: + if err := remapMessage( + &extensionRangeDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected extension range index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapServiceDescriptor( + serviceDescriptor *descriptorpb.ServiceDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case serviceMethodsTag: + if err := remapSlice( + &serviceDescriptor.Method, + remapNode, + remapMethodDescriptor, + ); err != nil { + return err + } + case serviceOptionsTag: + if err := remapMessage( + &serviceDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected service index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapMethodDescriptor( + methodDescriptor *descriptorpb.MethodDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) error { + for _, remapNode := range sourcePathRemaps { + switch remapNode.oldIndex { + case methodOptionsTag: + if err := remapMessage( + &methodDescriptor.Options, + remapNode, + remapOptionsDescriptor, + ); err != nil { + return err + } + default: + return fmt.Errorf("unexpected method index %d", remapNode.oldIndex) + } + } + return nil +} + +func remapOptionsDescriptor[T proto.Message]( + optionsMessage T, + sourcePathRemaps sourcePathsRemapTrie, +) error { + options := optionsMessage.ProtoReflect() + if !options.IsValid() { + return fmt.Errorf("invalid options %T", optionsMessage) + } + + // Create a mapping of fields to handle extensions. Extensions are not in the descriptor. + fields := make(map[int32]protoreflect.FieldDescriptor) + options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + fields[int32(fd.Number())] = fd + return true + }) + for _, remapNode := range sourcePathRemaps { + fd := fields[remapNode.oldIndex] + if fd == nil { + return fmt.Errorf("unexpected field number %d", remapNode.oldIndex) + } + if remapNode.newIndex != -1 { + return fmt.Errorf("unexpected options move %d to %d", remapNode.oldIndex, remapNode.newIndex) + } + options.Clear(fd) + } + return nil +} + +func remapMessage[T proto.Message]( + ptr *T, + remapNode *sourcePathsRemapTrieNode, + updateFn func(T, sourcePathsRemapTrie) error, +) error { + var zero T + if remapNode.newIndex == -1 { + // Remove the message. + *ptr = zero + return nil + } + message := shallowClone(*ptr) + if !message.ProtoReflect().IsValid() { + *ptr = zero + return nil // Nothing to update. + } + if remapNode.oldIndex != remapNode.newIndex { + return fmt.Errorf("unexpected message move %d to %d", remapNode.oldIndex, remapNode.newIndex) + } + if err := updateFn(message, remapNode.children); err != nil { + return err + } + *ptr = message + return nil +} + +func remapSlice[T proto.Message]( + slice *[]T, + remapNode *sourcePathsRemapTrieNode, + updateFn func(T, sourcePathsRemapTrie) error, +) error { + if remapNode.newIndex == -1 { + // Remove the slice. + *slice = nil + return nil + } + *slice = slices.Clone(*slice) // Shallow clone + for _, child := range remapNode.children { + if child.oldIndex != child.newIndex { + // TODO: add support for deleting and moving elements. + // If children are present, the element must be copied. + // Otherwise the element can only be moved. + return fmt.Errorf("unexpected child slice move %d to %d", child.oldIndex, child.newIndex) + } + index := child.oldIndex + item := shallowClone((*slice)[index]) + if err := updateFn(item, child.children); err != nil { + return err + } + (*slice)[index] = item + } + return nil +} + +func shallowClone[T proto.Message](src T) T { + sm := src.ProtoReflect() + dm := sm.New() + sm.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + dm.Set(fd, v) + return true + }) + return dm.Interface().(T) +} diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go b/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go new file mode 100644 index 0000000000..4805269d82 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go @@ -0,0 +1,232 @@ +// Copyright 2020-2025 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimageutil + +import ( + "bytes" + "context" + "sort" + "testing" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/slogtestext" + "github.com/bufbuild/buf/private/pkg/storage" + "github.com/bufbuild/buf/private/pkg/storage/storageos" + "github.com/jhump/protoreflect/v2/protoprint" + "github.com/stretchr/testify/require" + "golang.org/x/tools/txtar" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" +) + +func TestExcludeOptions(t *testing.T) { + t.Parallel() + + t.Run("NoneExcluded", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + ) + }) + t.Run("ExcludeMessage", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + "message_foo", "message_bar", "message_baz", + ) + }) + t.Run("ExcludeFoo", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + "message_foo", + "field_foo", + "oneof_foo", + "enum_foo", + "enum_value_foo", + "service_foo", + "method_foo", + "UsedOption.file_foo", + ) + }) + t.Run("OnlyFile", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + ) + }) + t.Run("OnlyOneOf", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + ) + }) + t.Run("OnlyEnumValue", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + ) + }) + t.Run("ExcludeAll", func(t *testing.T) { + t.Parallel() + testExcludeOptions( + t, "testdata/excludeoptions", + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + ) + }) +} + +func TestExcludeOptionImports(t *testing.T) { + t.Parallel() + + // This checks that when excluding options the imports are correctly dropped. + // For this case when both options are removed only a.proto should be left. + testdataDir := "testdata/excludeoptionimports" + bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) + require.NoError(t, err) + testModuleData := []bufmoduletesting.ModuleData{ + { + Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("a.proto")), + }, + { + Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("options.proto")), + NotTargeted: true, + }, + } + moduleSet, err := bufmoduletesting.NewModuleSet(testModuleData...) + require.NoError(t, err) + + // Safe to filter the image concurrently as its not being modified. + image, err := bufimage.BuildImage( + context.Background(), + slogtestext.NewLogger(t), + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), + bufimage.WithExcludeSourceCodeInfo(), + ) + require.NoError(t, err) + + t.Run("NoneExcluded", func(t *testing.T) { + t.Parallel() + testExcludeOptionsForImage(t, bucket, image) + }) + t.Run("ExcludeFoo", func(t *testing.T) { + t.Parallel() + testExcludeOptionsForImage(t, bucket, image, "message_foo") + }) + t.Run("ExcludeFooBar", func(t *testing.T) { + t.Parallel() + testExcludeOptionsForImage(t, bucket, image, "message_foo", "message_bar") + }) + t.Run("ExcludeBar", func(t *testing.T) { + t.Parallel() + testExcludeOptionsForImage(t, bucket, image, "message_bar") + }) +} + +func testExcludeOptions(t *testing.T, testdataDir string, options ...string) { + bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) + require.NoError(t, err) + testExcludeOptionsForModuleData(t, bucket, nil, options...) +} + +func testExcludeOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucket, moduleData []bufmoduletesting.ModuleData, options ...string) { + ctx := context.Background() + if len(moduleData) == 0 { + moduleData = append(moduleData, bufmoduletesting.ModuleData{ + Bucket: bucket, + }) + } + moduleSet, err := bufmoduletesting.NewModuleSet(moduleData...) + require.NoError(t, err) + + image, err := bufimage.BuildImage( + ctx, + slogtestext.NewLogger(t), + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), + bufimage.WithExcludeSourceCodeInfo(), + ) + require.NoError(t, err) + + testExcludeOptionsForImage(t, bucket, image, options...) +} + +func testExcludeOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, image bufimage.Image, options ...string) { + ctx := context.Background() + filteredImage, err := ExcludeOptions(image, options...) + require.NoError(t, err) + + files, err := protodesc.NewFiles(&descriptorpb.FileDescriptorSet{ + File: slicesext.Map(filteredImage.Files(), func(imageFile bufimage.ImageFile) *descriptorpb.FileDescriptorProto { + return imageFile.FileDescriptorProto() + }), + }) + require.NoError(t, err) + + archive := &txtar.Archive{} + printer := protoprint.Printer{ + SortElements: true, + Compact: true, + } + files.RangeFiles(func(fileDescriptor protoreflect.FileDescriptor) bool { + fileBuilder := &bytes.Buffer{} + require.NoError(t, printer.PrintProtoFile(fileDescriptor, fileBuilder), "expected no error while printing %q", fileDescriptor.Path()) + archive.Files = append( + archive.Files, + txtar.File{ + Name: fileDescriptor.Path(), + Data: fileBuilder.Bytes(), + }, + ) + return true + }) + sort.SliceStable(archive.Files, func(i, j int) bool { + return archive.Files[i].Name < archive.Files[j].Name + }) + generated := txtar.Format(archive) + expectedFile := t.Name() + ".txtar" + checkExpectation(t, ctx, generated, bucket, expectedFile) +} diff --git a/private/bufpkg/bufimage/bufimageutil/tags.go b/private/bufpkg/bufimage/bufimageutil/tags.go index 66d91c31c6..cf698a6f7b 100644 --- a/private/bufpkg/bufimage/bufimageutil/tags.go +++ b/private/bufpkg/bufimage/bufimageutil/tags.go @@ -26,14 +26,23 @@ const ( fileEnumsTag = 5 fileServicesTag = 6 fileExtensionsTag = 7 + fileOptionsTag = 8 messageFieldsTag = 2 messageNestedMessagesTag = 3 messageEnumsTag = 4 messageExtensionsTag = 6 + messageOptionsTag = 7 messageOneofsTag = 8 messageExtensionRangesTag = 5 messageReservedRangesTag = 9 messageReservedNamesTag = 10 + extensionRangeOptionsTag = 3 + fieldOptionsTag = 8 + oneofOptionsTag = 2 + enumOptionsTag = 3 enumValuesTag = 2 + enumValueOptionsTag = 3 serviceMethodsTag = 2 + serviceOptionsTag = 3 + methodOptionsTag = 4 ) diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeBar.txtar new file mode 100644 index 0000000000..88e0fe3290 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeBar.txtar @@ -0,0 +1,485 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "options.proto"; +message Foo { + option (message_foo) = { foo: "str" }; + Bar nested_bar = 2; + message Bar { + string bar = 1; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message OptionBar { + string bar = 1; +} +message OptionFoo { + string foo = 1; +} +extend google.protobuf.MessageOptions { + OptionFoo message_foo = 50000; + OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFoo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFoo.txtar new file mode 100644 index 0000000000..0706f98c00 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFoo.txtar @@ -0,0 +1,485 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "options.proto"; +message Foo { + Bar nested_bar = 2; + message Bar { + option (message_bar) = { bar: "str" }; + string bar = 1; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message OptionBar { + string bar = 1; +} +message OptionFoo { + string foo = 1; +} +extend google.protobuf.MessageOptions { + OptionFoo message_foo = 50000; + OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFooBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFooBar.txtar new file mode 100644 index 0000000000..8209070bbc --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFooBar.txtar @@ -0,0 +1,9 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Foo { + Bar nested_bar = 2; + message Bar { + string bar = 1; + } +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/NoneExcluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/NoneExcluded.txtar new file mode 100644 index 0000000000..1003545b2f --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/NoneExcluded.txtar @@ -0,0 +1,486 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "options.proto"; +message Foo { + option (message_foo) = { foo: "str" }; + Bar nested_bar = 2; + message Bar { + option (message_bar) = { bar: "str" }; + string bar = 1; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message OptionBar { + string bar = 1; +} +message OptionFoo { + string foo = 1; +} +extend google.protobuf.MessageOptions { + OptionFoo message_foo = 50000; + OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/a.proto new file mode 100644 index 0000000000..4de73ce641 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/a.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package pkg; +import "options.proto"; + +message Foo { + option (message_foo).foo = "str"; + message Bar { + option (message_bar).bar = "str"; + string bar = 1; + } + Bar nested_bar = 2; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/options.proto b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/options.proto new file mode 100644 index 0000000000..e2ed89085a --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/options.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; + +message OptionFoo { + string foo = 1; +} +message OptionBar { + string bar = 1; +} + +extend google.protobuf.MessageOptions { + optional OptionFoo message_foo = 50000; + optional OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar new file mode 100644 index 0000000000..e71bbd516c --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar @@ -0,0 +1,546 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +message Empty { +} +message Foo { + optional uint64 foo = 1 [jstype = JS_STRING]; + oneof testOneof { + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1; + } + enum FooFooEnum { + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} +service FooService { + rpc Do ( Empty ) returns ( Empty ); + rpc DoNot ( Empty ) returns ( Empty ); +} +extend Foo { + optional string extension = 11; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar new file mode 100644 index 0000000000..9c7c035296 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar @@ -0,0 +1,557 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +message Empty { +} +message Foo { + option (message_baz) = "str"; + optional uint64 foo = 1 [jstype = JS_STRING, (field_baz) = "str"]; + oneof testOneof { + option (oneof_baz) = "str"; + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1 [(field_baz) = "str"]; + } + enum FooFooEnum { + option (enum_baz) = "str"; + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1 [(enum_value_baz) = "str"]; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + option (enum_baz) = "str"; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1 [(enum_value_baz) = "str"]; +} +service FooService { + option (service_baz) = "str"; + rpc Do ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + } + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + } +} +extend Foo { + optional string extension = 11 [(field_baz) = "str"]; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar new file mode 100644 index 0000000000..9c33d16124 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar @@ -0,0 +1,573 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; +message Empty { +} +message Foo { + optional uint64 foo = 1 [ + jstype = JS_STRING, + (field_baz) = "str", + (field_foo) = { foo: "str" } + ]; + oneof testOneof { + option (oneof_baz) = "str"; + option (oneof_foo) = { foo: "str" }; + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1 [(field_baz) = "str", (field_foo) = { foo: "str" }]; + } + enum FooFooEnum { + option (enum_baz) = "str"; + option (enum_foo) = { foo: "str" }; + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + option (enum_baz) = "str"; + option (enum_foo) = { foo: "str" }; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; +} +service FooService { + option (service_baz) = "str"; + option (service_foo) = { foo: "str" }; + rpc Do ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } +} +extend Foo { + optional string extension = 11 [(field_baz) = "str", (field_foo) = { foo: "str" }]; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar new file mode 100644 index 0000000000..bcd0962533 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar @@ -0,0 +1,575 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; +message Empty { +} +message Foo { + option (message_baz) = "str"; + option (message_foo) = { foo: "str" }; + optional uint64 foo = 1 [ + jstype = JS_STRING, + (field_baz) = "str", + (field_foo) = { foo: "str" } + ]; + oneof testOneof { + option (oneof_baz) = "str"; + option (oneof_foo) = { foo: "str" }; + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1 [(field_baz) = "str", (field_foo) = { foo: "str" }]; + } + enum FooFooEnum { + option (enum_baz) = "str"; + option (enum_foo) = { foo: "str" }; + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + option (enum_baz) = "str"; + option (enum_foo) = { foo: "str" }; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; +} +service FooService { + option (service_baz) = "str"; + option (service_foo) = { foo: "str" }; + rpc Do ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } +} +extend Foo { + optional string extension = 11 [(field_baz) = "str", (field_foo) = { foo: "str" }]; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar new file mode 100644 index 0000000000..f89d852f21 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar @@ -0,0 +1,553 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +message Empty { +} +message Foo { + optional uint64 foo = 1 [jstype = JS_STRING]; + oneof testOneof { + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1; + } + enum FooFooEnum { + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; +} +service FooService { + rpc Do ( Empty ) returns ( Empty ); + rpc DoNot ( Empty ) returns ( Empty ); +} +extend Foo { + optional string extension = 11; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar new file mode 100644 index 0000000000..8949934292 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar @@ -0,0 +1,549 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; +message Empty { +} +message Foo { + optional uint64 foo = 1 [jstype = JS_STRING]; + oneof testOneof { + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1; + } + enum FooFooEnum { + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} +service FooService { + rpc Do ( Empty ) returns ( Empty ); + rpc DoNot ( Empty ) returns ( Empty ); +} +extend Foo { + optional string extension = 11; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar new file mode 100644 index 0000000000..3e3df1624a --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar @@ -0,0 +1,549 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +message Empty { +} +message Foo { + optional uint64 foo = 1 [jstype = JS_STRING]; + oneof testOneof { + option (oneof_baz) = "str"; + option (oneof_foo) = { foo: "str" }; + string bar = 2; + bytes baz = 3; + } + optional Bar nested_bar = 4; + message Bar { + optional string bar = 1; + } + enum FooFooEnum { + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1; + } + extensions 10 to max; +} +enum FooEnum { + option deprecated = true; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} +service FooService { + rpc Do ( Empty ) returns ( Empty ); + rpc DoNot ( Empty ) returns ( Empty ); +} +extend Foo { + optional string extension = 11; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto new file mode 100644 index 0000000000..04f28c525b --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto @@ -0,0 +1,82 @@ +syntax = "proto2"; +package pkg; +import "options.proto"; + +option (UsedOption.file_foo).foo = "str"; +option (UsedOption.file_baz) = "str"; + +message Foo { + option (message_foo).foo = "str"; + option (message_baz) = "str"; + + optional uint64 foo = 1 [ + (field_foo).foo = "str", + (field_baz) = "str", + jstype = JS_STRING + ]; + + oneof testOneof { + option (oneof_foo).foo = "str"; + option (oneof_baz) = "str"; + + string bar = 2; + bytes baz = 3; + } + + enum FooFooEnum { + option (enum_foo).foo = "str"; + option (enum_baz) = "str"; + + FOOFOO_ENUM_X = 0; + FOOFOO_ENUM_Y = 1 [ + (enum_value_foo).foo = "str", + (enum_value_baz) = "str" + ]; + } + + message Bar { + optional string bar = 1 [ + (field_foo).foo = "str", + (field_baz) = "str" + ]; + } + optional Bar nested_bar = 4; + + extensions 10 to max; +} + +enum FooEnum { + option (enum_foo).foo = "str"; + option (enum_baz) = "str"; + option deprecated = true; + + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1 [ + (enum_value_foo).foo = "str", + (enum_value_baz) = "str" + ]; +} + +message Empty{} + +service FooService { + option (service_foo).foo = "str"; + option (service_baz) = "str"; + + rpc Do(Empty) returns (Empty) { + option (method_foo).foo = "str"; + option (method_baz) = "str"; + }; + + rpc DoNot(Empty) returns (Empty) { + option (method_foo).foo = "str"; + option (method_baz) = "str"; + }; +} + +extend Foo { + optional string extension = 11 [ + (field_foo).foo = "str", + (field_baz) = "str" + ]; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto new file mode 100644 index 0000000000..57676254cb --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; + +message UnusedOption { + string foo = 1; +} + +message UsedOption { + extend google.protobuf.FileOptions { + optional UsedOption file_foo = 50000; + optional UnusedOption file_bar = 50001; + optional string file_baz = 50002; + } + + string foo = 1; +} + +extend google.protobuf.MessageOptions { + optional UsedOption message_foo = 50000; + optional UnusedOption message_bar = 50001; + optional string message_baz = 50002; +} +extend google.protobuf.FieldOptions { + optional UsedOption field_foo = 50000; + optional UnusedOption field_bar = 50001; + optional string field_baz = 50002; +} +extend google.protobuf.OneofOptions { + optional UsedOption oneof_foo = 50000; + optional UnusedOption oneof_bar = 50001; + optional string oneof_baz = 50002; +} +extend google.protobuf.EnumOptions { + optional UsedOption enum_foo = 50000; + optional UnusedOption enum_bar = 50001; + optional string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + optional UsedOption enum_value_foo = 50000; + optional UnusedOption enum_value_bar = 50001; + optional string enum_value_baz = 50002; +} +extend google.protobuf.ServiceOptions { + optional UsedOption service_foo = 50000; + optional UnusedOption service_bar = 50001; + optional string service_baz = 50002; +} +extend google.protobuf.MethodOptions { + optional UsedOption method_foo = 50000; + optional UnusedOption method_bar = 50001; + optional string method_baz = 50002; +} + +message Files { + google.protobuf.FileDescriptorSet files = 1; +} From e2608aafbb3b430a76ca5f9c9a27b6dcbdcbadb8 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 7 Feb 2025 11:05:47 +0000 Subject: [PATCH 02/80] Fix lint issues and file iter --- .golangci.yml | 6 ++ .../bufimage/bufimageutil/exclude_options.go | 77 ++++++++++--------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a25e20c79e..c25e0bec9b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -363,6 +363,12 @@ issues: # parsing from an Image, the operation should be safe. path: private/bufpkg/bufimage/bufimageutil/bufimageutil.go text: "G115:" + - linters: + - gosec + # bufimageutil is handling images and converting loop indices to int32. Since it is + # parsing from an Image, the operation should be safe. + path: private/bufpkg/bufimage/bufimageutil/exclude_options.go + text: "G115:" - linters: - gosec # Bounds checks have been added with assertion statements to ensure safe int -> int32 diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options.go b/private/bufpkg/bufimage/bufimageutil/exclude_options.go index d60eb00185..f7cfdaa578 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options.go +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options.go @@ -17,6 +17,7 @@ package bufimageutil import ( "fmt" "slices" + "sort" "strings" "github.com/bufbuild/buf/private/bufpkg/bufimage" @@ -131,6 +132,10 @@ func (f *protoOptionsFilter) fileDescriptor( fileDescriptor *descriptorpb.FileDescriptorProto, filePath string, ) (*descriptorpb.FileDescriptorProto, error) { + f.sourcePathRemaps = f.sourcePathRemaps[:0] + for key := range f.requiredTypes { + delete(f.requiredTypes, key) + } if f.requiredTypes == nil { f.requiredTypes = make(map[protoreflect.FullName]struct{}) } @@ -172,49 +177,49 @@ func (f *protoOptionsFilter) fileDescriptor( } // Fix imports. if len(fileImports) != len(fileDescriptor.Dependency) { - i := 0 + indexTo := int32(0) dependencyPath := []int32{fileDependencyTag} dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) newFileDescriptor.Dependency = make([]string, 0, len(fileImports)) - for index, dependency := range fileDescriptor.Dependency { - path := append(dependencyPath, int32(index)) + for indexFrom, dependency := range fileDescriptor.Dependency { + path := append(dependencyPath, int32(indexFrom)) if _, ok := fileImports[dependency]; ok { newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, dependency) - dependencyChanges[index] = int32(i) - if i != index { - f.sourcePathRemaps.markMoved(path, int32(i)) + dependencyChanges[indexFrom] = indexTo + if indexTo != int32(indexFrom) { + f.sourcePathRemaps.markMoved(path, indexTo) } - i++ + indexTo++ } else { f.sourcePathRemaps.markDeleted(path) - dependencyChanges[index] = -1 + dependencyChanges[indexFrom] = -1 } } publicDependencyPath := []int32{filePublicDependencyTag} newFileDescriptor.PublicDependency = make([]int32, 0, len(fileDescriptor.PublicDependency)) - for index, publicDependency := range fileDescriptor.PublicDependency { - path := append(publicDependencyPath, int32(index)) - newPublicDependency := dependencyChanges[publicDependency] - if newPublicDependency == -1 { + for indexFrom, publicDependency := range fileDescriptor.PublicDependency { + path := append(publicDependencyPath, int32(indexFrom)) + indexTo := dependencyChanges[publicDependency] + if indexTo == -1 { f.sourcePathRemaps.markDeleted(path) } else { - newFileDescriptor.PublicDependency = append(newFileDescriptor.PublicDependency, newPublicDependency) - if newPublicDependency != int32(index) { - f.sourcePathRemaps.markMoved(path, newPublicDependency) + newFileDescriptor.PublicDependency = append(newFileDescriptor.PublicDependency, indexTo) + if indexTo != int32(indexFrom) { + f.sourcePathRemaps.markMoved(path, indexTo) } } } weakDependencyPath := []int32{fileWeakDependencyTag} newFileDescriptor.WeakDependency = make([]int32, 0, len(fileDescriptor.WeakDependency)) - for index, weakDependency := range fileDescriptor.WeakDependency { - path := append(weakDependencyPath, int32(index)) - newWeakDependency := dependencyChanges[weakDependency] - if newWeakDependency == -1 { + for indexFrom, weakDependency := range fileDescriptor.WeakDependency { + path := append(weakDependencyPath, int32(indexFrom)) + indexTo := dependencyChanges[weakDependency] + if indexTo == -1 { f.sourcePathRemaps.markDeleted(path) } else { - newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, newWeakDependency) - if newWeakDependency != int32(index) { - f.sourcePathRemaps.markMoved(path, newWeakDependency) + newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, indexTo) + if indexTo != int32(indexFrom) { + f.sourcePathRemaps.markMoved(path, indexTo) } } } @@ -243,11 +248,6 @@ func (f *protoOptionsFilter) fileDescriptor( Location: newLocations, } } - // Cleanup. - f.sourcePathRemaps = f.sourcePathRemaps[:0] - for key := range f.requiredTypes { - delete(f.requiredTypes, key) - } return newFileDescriptor, nil } @@ -664,22 +664,26 @@ func remapOptionsDescriptor[T proto.Message]( if !options.IsValid() { return fmt.Errorf("invalid options %T", optionsMessage) } - - // Create a mapping of fields to handle extensions. Extensions are not in the descriptor. - fields := make(map[int32]protoreflect.FieldDescriptor) + // Create a mapping of fieldDescriptors to handle extensions. This is required + // because extensions are not in the descriptor. + fieldDescriptors := make([]protoreflect.FieldDescriptor, len(sourcePathRemaps)) options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - fields[int32(fd.Number())] = fd + if index, found := sort.Find(len(sourcePathRemaps), func(i int) int { + return int(fd.Number()) - int(sourcePathRemaps[i].oldIndex) + }); found { + fieldDescriptors[index] = fd + } return true }) - for _, remapNode := range sourcePathRemaps { - fd := fields[remapNode.oldIndex] - if fd == nil { + for index, remapNode := range sourcePathRemaps { + fieldDescriptor := fieldDescriptors[index] + if fieldDescriptor == nil { return fmt.Errorf("unexpected field number %d", remapNode.oldIndex) } if remapNode.newIndex != -1 { return fmt.Errorf("unexpected options move %d to %d", remapNode.oldIndex, remapNode.newIndex) } - options.Clear(fd) + options.Clear(fieldDescriptor) } return nil } @@ -745,5 +749,6 @@ func shallowClone[T proto.Message](src T) T { dm.Set(fd, v) return true }) - return dm.Interface().(T) + value, _ := dm.Interface().(T) // Safe to assert. + return value } From d05d21074cd17f768ec57a4eec701b6bb444f7f3 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 12 Feb 2025 19:52:40 +0000 Subject: [PATCH 03/80] Use protoreflect for remapping --- .../bufimage/bufimageutil/exclude_options.go | 469 ++++-------------- 1 file changed, 109 insertions(+), 360 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options.go b/private/bufpkg/bufimage/bufimageutil/exclude_options.go index f7cfdaa578..33f436aae3 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options.go +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options.go @@ -155,13 +155,13 @@ func (f *protoOptionsFilter) fileDescriptor( } // Recursively apply the source path remaps. - newFileDescriptor := shallowClone(fileDescriptor) - if err := remapFileDescriptor( - newFileDescriptor, - f.sourcePathRemaps, - ); err != nil { + fileDescriptorMessage := fileDescriptor.ProtoReflect() + newFileDescriptorMessage, err := remapMessageReflect(fileDescriptorMessage, f.sourcePathRemaps) + if err != nil { return nil, err } + newFileDescriptor, _ := newFileDescriptorMessage.Interface().(*descriptorpb.FileDescriptorProto) + // Convert the required types to imports. fileImports := make(map[string]struct{}, len(fileDescriptor.Dependency)) for requiredType := range f.requiredTypes { @@ -357,398 +357,147 @@ func (f *protoOptionsFilter) addFieldType(field *descriptorpb.FieldDescriptorPro return nil } -// TODO: rewrite using protoreflect. -func remapFileDescriptor( - fileDescriptor *descriptorpb.FileDescriptorProto, +func remapMessageReflect( + message protoreflect.Message, sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case fileDependencyTag: - // Dependencies are handled after message remaps. - return fmt.Errorf("unexpected dependency move %d to %d", remapNode.oldIndex, remapNode.newIndex) - case filePublicDependencyTag: - return fmt.Errorf("unexpected public dependency move %d to %d", remapNode.oldIndex, remapNode.newIndex) - case fileWeakDependencyTag: - return fmt.Errorf("unexpected weak dependency move %d to %d", remapNode.oldIndex, remapNode.newIndex) - case fileMessagesTag: - if err := remapSlice( - &fileDescriptor.MessageType, - remapNode, - remapMessageDescriptor, - ); err != nil { - return err - } - case fileEnumsTag: - if err := remapSlice( - &fileDescriptor.EnumType, - remapNode, - remapEnumDescriptor, - ); err != nil { - return err - } - case fileServicesTag: - if err := remapSlice( - &fileDescriptor.Service, - remapNode, - remapServiceDescriptor, - ); err != nil { - return err - } - case fileExtensionsTag: - if err := remapSlice( - &fileDescriptor.Extension, - remapNode, - remapFieldDescriptor, - ); err != nil { - return err - } - case fileOptionsTag: - if err := remapMessage( - &fileDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err - } - default: - return fmt.Errorf("unexpected file index %d", remapNode.oldIndex) - } +) (protoreflect.Message, error) { + if len(sourcePathRemaps) == 0 { + return message, nil } - return nil -} - -func remapMessageDescriptor( - messageDescriptor *descriptorpb.DescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case messageFieldsTag: - if err := remapSlice( - &messageDescriptor.Field, - remapNode, - remapFieldDescriptor, - ); err != nil { - return err - } - case messageNestedMessagesTag: - if err := remapSlice( - &messageDescriptor.NestedType, - remapNode, - remapMessageDescriptor, - ); err != nil { - return err - } - case messageEnumsTag: - if err := remapSlice( - &messageDescriptor.EnumType, - remapNode, - remapEnumDescriptor, - ); err != nil { - return err - } - case messageExtensionsTag: - if err := remapSlice( - &messageDescriptor.Extension, - remapNode, - remapFieldDescriptor, - ); err != nil { - return err - } - case messageOptionsTag: - if err := remapMessage( - &messageDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err - } - case messageOneofsTag: - if err := remapSlice( - &messageDescriptor.OneofDecl, - remapNode, - remapOneofDescriptor, - ); err != nil { - return err - } - case messageExtensionRangesTag: - if err := remapSlice( - &messageDescriptor.ExtensionRange, - remapNode, - remapExtensionRangeDescriptor, - ); err != nil { - return err - } - case messageReservedRangesTag: - // TODO: handle reserved ranges tag. - return fmt.Errorf("unexpected reserved tags move %d to %d", remapNode.oldIndex, remapNode.newIndex) - case messageReservedNamesTag: - // TODO: handle reserved names. - return fmt.Errorf("unexpected reserved names move %d to %d", remapNode.oldIndex, remapNode.newIndex) - default: - return fmt.Errorf("unexpected message index %d", remapNode.oldIndex) - } + if !message.IsValid() { + return nil, fmt.Errorf("invalid message %T", message) } - return nil -} - -func remapFieldDescriptor( - fieldDescriptor *descriptorpb.FieldDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case fieldOptionsTag: - if err := remapMessage( - &fieldDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err - } - default: - return fmt.Errorf("unexpected field index %d", remapNode.oldIndex) - } + fieldDescriptors, err := getFieldDescriptors(message, sourcePathRemaps) + if err != nil { + return nil, err } - return nil -} - -func remapOneofDescriptor( - oneofDescriptor *descriptorpb.OneofDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case oneofOptionsTag: - if err := remapMessage( - &oneofDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err - } - default: - return fmt.Errorf("unexpected oneof index %d", remapNode.oldIndex) + message = shallowCloneReflect(message) + for index, remapNode := range sourcePathRemaps { + fieldDescriptor := fieldDescriptors[index] + if fieldDescriptor == nil { + return nil, fmt.Errorf("missing field descriptor %d on type %s", remapNode.oldIndex, message.Descriptor().FullName()) } - } - return nil -} - -func remapEnumDescriptor( - enumDescriptor *descriptorpb.EnumDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case enumValuesTag: - if err := remapSlice( - &enumDescriptor.Value, - remapNode, - remapEnumValueDescriptor, - ); err != nil { - return err + if remapNode.newIndex == -1 { + message.Clear(fieldDescriptor) + continue + } else if remapNode.newIndex != remapNode.oldIndex { + return nil, fmt.Errorf("unexpected field move %d to %d", remapNode.oldIndex, remapNode.newIndex) + } + value := message.Get(fieldDescriptor) + switch { + case fieldDescriptor.IsList(): + if len(remapNode.children) == 0 { + break } - case enumOptionsTag: - if err := remapMessage( - &enumDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err + newList := message.NewField(fieldDescriptor).List() + if err := remapListReflect(newList, value.List(), remapNode.children); err != nil { + return nil, err } + value = protoreflect.ValueOfList(newList) + case fieldDescriptor.IsMap(): + panic("map fields not yet supported") default: - return fmt.Errorf("unexpected enum index %d", remapNode.oldIndex) - } - } - return nil -} - -func remapEnumValueDescriptor( - enumValueDescriptor *descriptorpb.EnumValueDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case enumValueOptionsTag: - if err := remapMessage( - &enumValueDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err + fieldMessage, err := remapMessageReflect(value.Message(), remapNode.children) + if err != nil { + return nil, err } - default: - return fmt.Errorf("unexpected enum value index %d", remapNode.oldIndex) + value = protoreflect.ValueOfMessage(fieldMessage) } + message.Set(fieldDescriptor, value) } - return nil + return message, nil } -func remapExtensionRangeDescriptor( - extensionRangeDescriptor *descriptorpb.DescriptorProto_ExtensionRange, +func remapListReflect( + dstList protoreflect.List, + srcList protoreflect.List, sourcePathRemaps sourcePathsRemapTrie, ) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case extensionRangeOptionsTag: - if err := remapMessage( - &extensionRangeDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err - } - default: - return fmt.Errorf("unexpected extension range index %d", remapNode.oldIndex) - } + if len(sourcePathRemaps) == 0 { + return nil } - return nil -} - -func remapServiceDescriptor( - serviceDescriptor *descriptorpb.ServiceDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case serviceMethodsTag: - if err := remapSlice( - &serviceDescriptor.Method, - remapNode, - remapMethodDescriptor, - ); err != nil { - return err - } - case serviceOptionsTag: - if err := remapMessage( - &serviceDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { - return err + toIndex := 0 + sourcePathIndex := 0 + for fromIndex := 0; fromIndex < srcList.Len(); fromIndex++ { + var remapNode *sourcePathsRemapTrieNode + for ; sourcePathIndex < len(sourcePathRemaps); sourcePathIndex++ { + nextRemapNode := sourcePathRemaps[sourcePathIndex] + if index := int(nextRemapNode.oldIndex); index > fromIndex { + break + } else if index == fromIndex { + remapNode = nextRemapNode + break } - default: - return fmt.Errorf("unexpected service index %d", remapNode.oldIndex) } - } - return nil -} - -func remapMethodDescriptor( - methodDescriptor *descriptorpb.MethodDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) error { - for _, remapNode := range sourcePathRemaps { - switch remapNode.oldIndex { - case methodOptionsTag: - if err := remapMessage( - &methodDescriptor.Options, - remapNode, - remapOptionsDescriptor, - ); err != nil { + value := srcList.Get(fromIndex) + if remapNode == nil { + dstList.Append(value) + toIndex++ + continue + } + if remapNode.newIndex == -1 { + continue + } + if fromIndex != int(remapNode.oldIndex) || toIndex != int(remapNode.newIndex) { + return fmt.Errorf("unexpected list move %d to %d, expected %d to %d", remapNode.oldIndex, remapNode.newIndex, fromIndex, toIndex) + } + // If no children, the value is unchanged. + if len(remapNode.children) > 0 { + // Must be a list of messages to have children. + indexMessage, err := remapMessageReflect(value.Message(), remapNode.children) + if err != nil { return err } - default: - return fmt.Errorf("unexpected method index %d", remapNode.oldIndex) + value = protoreflect.ValueOfMessage(indexMessage) } + dstList.Append(value) + toIndex++ } return nil } -func remapOptionsDescriptor[T proto.Message]( - optionsMessage T, +func getFieldDescriptors( + message protoreflect.Message, sourcePathRemaps sourcePathsRemapTrie, -) error { - options := optionsMessage.ProtoReflect() - if !options.IsValid() { - return fmt.Errorf("invalid options %T", optionsMessage) - } - // Create a mapping of fieldDescriptors to handle extensions. This is required - // because extensions are not in the descriptor. +) ([]protoreflect.FieldDescriptor, error) { + var hasExtension bool fieldDescriptors := make([]protoreflect.FieldDescriptor, len(sourcePathRemaps)) - options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - if index, found := sort.Find(len(sourcePathRemaps), func(i int) int { - return int(fd.Number()) - int(sourcePathRemaps[i].oldIndex) - }); found { - fieldDescriptors[index] = fd - } - return true - }) + fields := message.Descriptor().Fields() for index, remapNode := range sourcePathRemaps { - fieldDescriptor := fieldDescriptors[index] + fieldDescriptor := fields.ByNumber(protoreflect.FieldNumber(remapNode.oldIndex)) if fieldDescriptor == nil { - return fmt.Errorf("unexpected field number %d", remapNode.oldIndex) - } - if remapNode.newIndex != -1 { - return fmt.Errorf("unexpected options move %d to %d", remapNode.oldIndex, remapNode.newIndex) + hasExtension = true + } else { + fieldDescriptors[index] = fieldDescriptor } - options.Clear(fieldDescriptor) - } - return nil -} - -func remapMessage[T proto.Message]( - ptr *T, - remapNode *sourcePathsRemapTrieNode, - updateFn func(T, sourcePathsRemapTrie) error, -) error { - var zero T - if remapNode.newIndex == -1 { - // Remove the message. - *ptr = zero - return nil - } - message := shallowClone(*ptr) - if !message.ProtoReflect().IsValid() { - *ptr = zero - return nil // Nothing to update. } - if remapNode.oldIndex != remapNode.newIndex { - return fmt.Errorf("unexpected message move %d to %d", remapNode.oldIndex, remapNode.newIndex) + if !hasExtension { + return fieldDescriptors, nil } - if err := updateFn(message, remapNode.children); err != nil { - return err - } - *ptr = message - return nil -} - -func remapSlice[T proto.Message]( - slice *[]T, - remapNode *sourcePathsRemapTrieNode, - updateFn func(T, sourcePathsRemapTrie) error, -) error { - if remapNode.newIndex == -1 { - // Remove the slice. - *slice = nil - return nil - } - *slice = slices.Clone(*slice) // Shallow clone - for _, child := range remapNode.children { - if child.oldIndex != child.newIndex { - // TODO: add support for deleting and moving elements. - // If children are present, the element must be copied. - // Otherwise the element can only be moved. - return fmt.Errorf("unexpected child slice move %d to %d", child.oldIndex, child.newIndex) + message.Range(func(fieldDescriptor protoreflect.FieldDescriptor, _ protoreflect.Value) bool { + if !fieldDescriptor.IsExtension() { + return true // Skip non-extension fields. } - index := child.oldIndex - item := shallowClone((*slice)[index]) - if err := updateFn(item, child.children); err != nil { - return err + if index, found := sort.Find(len(sourcePathRemaps), func(i int) int { + return int(fieldDescriptor.Number()) - int(sourcePathRemaps[i].oldIndex) + }); found { + fieldDescriptors[index] = fieldDescriptor } - (*slice)[index] = item - } - return nil + return true + }) + return fieldDescriptors, nil } func shallowClone[T proto.Message](src T) T { - sm := src.ProtoReflect() - dm := sm.New() - sm.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - dm.Set(fd, v) + value, _ := shallowCloneReflect(src.ProtoReflect()).Interface().(T) // Safe to assert. + return value +} + +func shallowCloneReflect(src protoreflect.Message) protoreflect.Message { + dst := src.New() + src.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + dst.Set(fd, v) return true }) - value, _ := dm.Interface().(T) // Safe to assert. - return value + return dst } From 5940c75bf83ec6ae393f11e0a50c49575d59d498 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 12 Feb 2025 20:32:57 +0000 Subject: [PATCH 04/80] Remap dependencies --- .../bufimage/bufimageutil/exclude_options.go | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options.go b/private/bufpkg/bufimage/bufimageutil/exclude_options.go index 33f436aae3..110edca338 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options.go +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options.go @@ -155,12 +155,6 @@ func (f *protoOptionsFilter) fileDescriptor( } // Recursively apply the source path remaps. - fileDescriptorMessage := fileDescriptor.ProtoReflect() - newFileDescriptorMessage, err := remapMessageReflect(fileDescriptorMessage, f.sourcePathRemaps) - if err != nil { - return nil, err - } - newFileDescriptor, _ := newFileDescriptorMessage.Interface().(*descriptorpb.FileDescriptorProto) // Convert the required types to imports. fileImports := make(map[string]struct{}, len(fileDescriptor.Dependency)) @@ -180,11 +174,9 @@ func (f *protoOptionsFilter) fileDescriptor( indexTo := int32(0) dependencyPath := []int32{fileDependencyTag} dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) - newFileDescriptor.Dependency = make([]string, 0, len(fileImports)) for indexFrom, dependency := range fileDescriptor.Dependency { path := append(dependencyPath, int32(indexFrom)) if _, ok := fileImports[dependency]; ok { - newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, dependency) dependencyChanges[indexFrom] = indexTo if indexTo != int32(indexFrom) { f.sourcePathRemaps.markMoved(path, indexTo) @@ -196,35 +188,33 @@ func (f *protoOptionsFilter) fileDescriptor( } } publicDependencyPath := []int32{filePublicDependencyTag} - newFileDescriptor.PublicDependency = make([]int32, 0, len(fileDescriptor.PublicDependency)) for indexFrom, publicDependency := range fileDescriptor.PublicDependency { path := append(publicDependencyPath, int32(indexFrom)) indexTo := dependencyChanges[publicDependency] if indexTo == -1 { f.sourcePathRemaps.markDeleted(path) - } else { - newFileDescriptor.PublicDependency = append(newFileDescriptor.PublicDependency, indexTo) - if indexTo != int32(indexFrom) { - f.sourcePathRemaps.markMoved(path, indexTo) - } + } else if indexTo != int32(indexFrom) { + f.sourcePathRemaps.markMoved(path, indexTo) } } weakDependencyPath := []int32{fileWeakDependencyTag} - newFileDescriptor.WeakDependency = make([]int32, 0, len(fileDescriptor.WeakDependency)) for indexFrom, weakDependency := range fileDescriptor.WeakDependency { path := append(weakDependencyPath, int32(indexFrom)) indexTo := dependencyChanges[weakDependency] if indexTo == -1 { f.sourcePathRemaps.markDeleted(path) - } else { - newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, indexTo) - if indexTo != int32(indexFrom) { - f.sourcePathRemaps.markMoved(path, indexTo) - } + } else if indexTo != int32(indexFrom) { + f.sourcePathRemaps.markMoved(path, indexTo) } } } // Remap the source code info. + fileDescriptorMessage := fileDescriptor.ProtoReflect() + newFileDescriptorMessage, err := remapMessageReflect(fileDescriptorMessage, f.sourcePathRemaps) + if err != nil { + return nil, err + } + newFileDescriptor, _ := newFileDescriptorMessage.Interface().(*descriptorpb.FileDescriptorProto) if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { newLocations := make([]*descriptorpb.SourceCodeInfo_Location, 0, len(locations)) for _, location := range locations { From f763966b0543634e404d401b157b91dd30724427 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 14 Feb 2025 10:27:17 +0000 Subject: [PATCH 05/80] Refactor for image filtering --- .../bufimage/bufimageutil/bufimageutil.go | 91 ++- .../bufimage/bufimageutil/exclude_options.go | 736 ++++++++++++++---- .../bufimageutil/exclude_options_test.go | 146 ++-- .../bufimage/bufimageutil/image_index.go | 2 + .../bufimage/bufimageutil/image_types.go | 305 ++++++++ .../bufimageutil/testdata/filtertypes/a.proto | 31 + .../bufimageutil/testdata/filtertypes/b.proto | 0 .../nesting/TestFilterTypes/ExcludeBar.txtar | 22 + .../nesting/TestFilterTypes/IncludeBar.txtar | 23 + 9 files changed, 1148 insertions(+), 208 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimageutil/image_types.go create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/b.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index bc019e4f22..fe36c552cb 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -23,8 +23,6 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/protoplugin/protopluginutil" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) @@ -99,6 +97,53 @@ func WithAllowFilterByImportedType() ImageFilterOption { } } +func WithIncludeTypes(typeNames ...string) ImageFilterOption { + return func(opts *imageFilterOptions) { + if opts.includeTypes == nil { + opts.includeTypes = make(map[string]struct{}, len(typeNames)) + } + for _, typeName := range typeNames { + opts.includeTypes[typeName] = struct{}{} + } + } +} + +func WithExcludeTypes(typeNames ...string) ImageFilterOption { + return func(opts *imageFilterOptions) { + if opts.excludeTypes == nil { + opts.excludeTypes = make(map[string]struct{}, len(typeNames)) + } + for _, typeName := range typeNames { + opts.excludeTypes[typeName] = struct{}{} + } + //opts.excludeTypes = append(opts.excludeTypes, typeNames...) + } +} + +func WithIncludeOptions(typeNames ...string) ImageFilterOption { + return func(opts *imageFilterOptions) { + if opts.includeOptions == nil { + opts.includeOptions = make(map[string]struct{}, len(typeNames)) + } + for _, typeName := range typeNames { + opts.includeOptions[typeName] = struct{}{} + } + //opts.includeOptions = append(opts.includeOptions, typeNames...) + } +} + +func WithExcludeOptions(typeNames ...string) ImageFilterOption { + return func(opts *imageFilterOptions) { + if opts.excludeOptions == nil { + opts.excludeOptions = make(map[string]struct{}, len(typeNames)) + } + for _, typeName := range typeNames { + opts.excludeOptions[typeName] = struct{}{} + } + //opts.excludeOptions = append(opts.excludeOptions, typeNames...) + } +} + // ImageFilteredByTypes returns a minimal image containing only the descriptors // required to define those types. The resulting contains only files in which // those descriptors and their transitive closure of required descriptors, with @@ -171,11 +216,23 @@ func ImageFilteredByTypes(image bufimage.Image, types ...string) (bufimage.Image return ImageFilteredByTypesWithOptions(image, types) } +func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.Image, error) { + if len(options) == 0 { + return image, nil + } + filterOptions := newImageFilterOptions() + for _, option := range options { + option(filterOptions) + } + return filterImage(image, filterOptions) +} + // ImageFilteredByTypesWithOptions returns a minimal image containing only the descriptors // required to define those types. See ImageFilteredByTypes for more details. This version // allows for customizing the behavior with options. func ImageFilteredByTypesWithOptions(image bufimage.Image, types []string, opts ...ImageFilterOption) (bufimage.Image, error) { - options := newImageFilterOptions() + return FilterImage(image, append(opts, WithIncludeTypes(types...))...) + /*options := newImageFilterOptions() for _, o := range opts { o(options) } @@ -366,7 +423,7 @@ func ImageFilteredByTypesWithOptions(image bufimage.Image, types []string, opts imageFileDescriptor.SourceCodeInfo.Location = imageFileDescriptor.SourceCodeInfo.Location[:i] } } - return bufimage.NewImage(includedFiles) + return bufimage.NewImage(includedFiles)*/ } // StripSourceRetentionOptions strips any options with a retention of "source" from @@ -384,6 +441,7 @@ func StripSourceRetentionOptions(image bufimage.Image) (bufimage.Image, error) { return bufimage.NewImage(updatedFiles) } +/* // trimMessageDescriptors removes (nested) messages and nested enums from a slice // of message descriptors if their type names are not found in the toKeep map. func trimMessageDescriptors( @@ -452,7 +510,7 @@ func trimSlice[T namedDescriptor]( } return in[:i] } - +/* // transitiveClosure accumulates the elements, files, and needed imports for a // subset of an image. When an element is added to the closure, all of its // dependencies are recursively added. @@ -612,7 +670,7 @@ func (t *transitiveClosure) addElement( case *descriptorpb.MethodDescriptorProto: inputName := strings.TrimPrefix(typedDescriptor.GetInputType(), ".") - inputDescriptor, ok := imageIndex.ByName[inputName] + inputDescriptor, ok := imageIndex.NameToDescriptor[inputName] if !ok { return fmt.Errorf("missing %q", inputName) } @@ -621,7 +679,7 @@ func (t *transitiveClosure) addElement( } outputName := strings.TrimPrefix(typedDescriptor.GetOutputType(), ".") - outputDescriptor, ok := imageIndex.ByName[outputName] + outputDescriptor, ok := imageIndex.NameToDescriptor[outputName] if !ok { return fmt.Errorf("missing %q", outputName) } @@ -639,7 +697,7 @@ func (t *transitiveClosure) addElement( return fmt.Errorf("expected extendee for field %q to not be empty", descriptorInfo.fullName) } extendeeName := strings.TrimPrefix(typedDescriptor.GetExtendee(), ".") - extendeeDescriptor, ok := imageIndex.ByName[extendeeName] + extendeeDescriptor, ok := imageIndex.NameToDescriptor[extendeeName] if !ok { return fmt.Errorf("missing %q", extendeeName) } @@ -714,7 +772,7 @@ func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProt descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, descriptorpb.FieldDescriptorProto_TYPE_GROUP: typeName := strings.TrimPrefix(field.GetTypeName(), ".") - typeDescriptor, ok := imageIndex.ByName[typeName] + typeDescriptor, ok := imageIndex.NameToDescriptor[typeName] if !ok { return fmt.Errorf("missing %q", typeName) } @@ -885,7 +943,7 @@ func (t *transitiveClosure) exploreOptionSingularValueForAny( typeURL := msg.Get(typeURLFd).String() pos := strings.LastIndexByte(typeURL, '/') msgType := typeURL[pos+1:] - d, _ := imageIndex.ByName[msgType].(*descriptorpb.DescriptorProto) + d, _ := imageIndex.NameToDescriptor[msgType].(*descriptorpb.DescriptorProto) if d != nil { if err := t.addElement(d, referrerFile, false, imageIndex, opts); err != nil { return err @@ -901,7 +959,7 @@ func (t *transitiveClosure) exploreOptionSingularValueForAny( return err == nil }) return err -} +}*/ type int32Range struct { start, end int32 // both inclusive @@ -981,6 +1039,11 @@ type imageFilterOptions struct { includeCustomOptions bool includeKnownExtensions bool allowImportedTypes bool + + includeTypes map[string]struct{} + excludeTypes map[string]struct{} + includeOptions map[string]struct{} + excludeOptions map[string]struct{} } func newImageFilterOptions() *imageFilterOptions { @@ -992,10 +1055,14 @@ func newImageFilterOptions() *imageFilterOptions { } func stripSourceRetentionOptionsFromFile(imageFile bufimage.ImageFile) (bufimage.ImageFile, error) { - updatedFileDescriptor, err := protopluginutil.StripSourceRetentionOptions(imageFile.FileDescriptorProto()) + fileDescriptor := imageFile.FileDescriptorProto() + updatedFileDescriptor, err := protopluginutil.StripSourceRetentionOptions(fileDescriptor) if err != nil { return nil, err } + if updatedFileDescriptor == fileDescriptor { + return imageFile, nil + } return bufimage.NewImageFile( updatedFileDescriptor, imageFile.FullName(), diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options.go b/private/bufpkg/bufimage/bufimageutil/exclude_options.go index 110edca338..2db0dc8003 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options.go +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options.go @@ -15,51 +15,45 @@ package bufimageutil import ( + "encoding/json" "fmt" "slices" "sort" "strings" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/protoencoding" - "github.com/bufbuild/protocompile/walk" + "github.com/bufbuild/buf/private/pkg/syserror" "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) -// ExcludeOptions returns an Image that omits all provided options. -// -// If the Image has none of the provided options, the original Image is returned. -// If the Image has any of the provided options, a new Image is returned. -// If an import is no longer imported by any file, it is removed from the Image. -// -// The returned Image will be a shallow copy of the original Image. -// The FileDescriptors in the returned Image.Files will be a shallow copy of the original FileDescriptorProto. -// -// The options are specified by their type name, e.g. "buf.validate.message". -func ExcludeOptions(image bufimage.Image, options ...string) (bufimage.Image, error) { - if len(options) == 0 { - return image, nil +// filterImage filters the Image for the given options. +func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Image, error) { + imageTypeIndex, err := newImageTypeIndex(image) + if err != nil { + return nil, err } - imageFiles := image.Files() - newImageFiles := make([]bufimage.ImageFile, 0, len(image.Files())) - importsByFilePath := make(map[string]struct{}) - - optionsByName := make(map[string]struct{}, len(options)) - for _, option := range options { - optionsByName[option] = struct{}{} + b, _ := json.MarshalIndent(imageTypeIndex.TypeSet, "", " ") + fmt.Println("index:", string(b)) + typeFilter, err := createTypeFilter(imageTypeIndex, options) + if err != nil { + return nil, err } - optionsFilter := func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool { - _, ok := optionsByName[string(fd.FullName())] - return !ok + fmt.Println("\ttypeFilter:", typeFilter) + fmt.Println("\t\tincludes:", typeFilter.include) + fmt.Println("\t\texcludes:", typeFilter.exclude) + optionsFilter, err := createOptionsFilter(imageTypeIndex, options) + if err != nil { + return nil, err } - filter := newOptionsFilter(image.Resolver(), optionsFilter) // Loop over image files in revserse DAG order. Imports that are no longer // imported by a previous file are dropped from the image. + imageFiles := image.Files() dirty := false + newImageFiles := make([]bufimage.ImageFile, 0, len(image.Files())) + importsByFilePath := make(map[string]struct{}) for i := len(image.Files()) - 1; i >= 0; i-- { imageFile := imageFiles[i] imageFilePath := imageFile.Path() @@ -69,107 +63,129 @@ func ExcludeOptions(image bufimage.Image, options ...string) (bufimage.Image, er continue } } - oldFileDescriptor := imageFile.FileDescriptorProto() - newFileDescriptor, err := filter.fileDescriptor(oldFileDescriptor, imageFilePath) + newImageFile, err := filterImageFile( + imageFile, + imageTypeIndex, + typeFilter, + optionsFilter, + ) if err != nil { return nil, err } - for _, filePath := range newFileDescriptor.Dependency { - importsByFilePath[filePath] = struct{}{} + dirty = dirty || newImageFile != imageFile + if newImageFile == nil { + continue // Filtered out. } - // Create a new image file with the modified file descriptor. - if newFileDescriptor != oldFileDescriptor { - dirty = true - imageFile, err = bufimage.NewImageFile( - newFileDescriptor, - imageFile.FullName(), - imageFile.CommitID(), - imageFile.ExternalPath(), - imageFile.LocalPath(), - imageFile.IsImport(), - imageFile.IsSyntaxUnspecified(), - imageFile.UnusedDependencyIndexes(), - ) - if err != nil { - return nil, err - } + for _, filePath := range newImageFile.FileDescriptorProto().Dependency { + importsByFilePath[filePath] = struct{}{} } - newImageFiles = append(newImageFiles, imageFile) + newImageFiles = append(newImageFiles, newImageFile) } if dirty { - // Revserse the image files back to DAG order. + // Reverse the image files back to DAG order. slices.Reverse(newImageFiles) return bufimage.NewImage(newImageFiles) } return image, nil } -// protoOptionsFilter is a filter for options. -// -// This filter is applied to FileDescriptorProto to remove options. A new shallow -// copy of the FileDescriptorProto is returned with the options removed. If no -// options are removed, the original FileDescriptorProto is returned. -type protoOptionsFilter struct { - // resolver is used to resvole parent files for required types. - resolver protodesc.Resolver - // optionsFilter returns true if the option should be kept. - optionsFilter func(protoreflect.FieldDescriptor, protoreflect.Value) bool - - // sourcePathsRemaps is a trie of source path remaps. Reset per file iteration. - sourcePathRemaps sourcePathsRemapTrie - // requiredTypes is a set of required types. Reset per file iteration. - requiredTypes map[protoreflect.FullName]struct{} +func filterImageFile( + imageFile bufimage.ImageFile, + imageTypeIndex *imageTypeIndex, + typesFilter fullNameFilter, + optionsFilter fullNameFilter, +) (bufimage.ImageFile, error) { + fileDescriptor := imageFile.FileDescriptorProto() + var sourcePathsRemap sourcePathsRemapTrie + isIncluded, err := addRemapsForFileDescriptor( + &sourcePathsRemap, + fileDescriptor, + imageTypeIndex, + typesFilter, + optionsFilter, + ) + if err != nil { + return nil, err + } + if !isIncluded { + return nil, nil // Filtered out. + } + if len(sourcePathsRemap) == 0 { + return imageFile, nil // No changes required. + } + newFileDescriptor, err := remapFileDescriptor(fileDescriptor, sourcePathsRemap) + if err != nil { + return nil, err + } + return bufimage.NewImageFile( + newFileDescriptor, + imageFile.FullName(), + imageFile.CommitID(), + imageFile.ExternalPath(), + imageFile.LocalPath(), + imageFile.IsImport(), + imageFile.IsSyntaxUnspecified(), + imageFile.UnusedDependencyIndexes(), + ) } -func newOptionsFilter(resolver protoencoding.Resolver, optionsFilter func(protoreflect.FieldDescriptor, protoreflect.Value) bool) *protoOptionsFilter { - return &protoOptionsFilter{ - resolver: resolver, - optionsFilter: optionsFilter, - } +type sourcePathsBuilder struct { + filePath string + imageTypeIndex *imageTypeIndex + typesFilter fullNameFilter + optionsFilter fullNameFilter + fileImports map[string]struct{} } -func (f *protoOptionsFilter) fileDescriptor( +func addRemapsForFileDescriptor( + sourcePathsRemap *sourcePathsRemapTrie, fileDescriptor *descriptorpb.FileDescriptorProto, - filePath string, -) (*descriptorpb.FileDescriptorProto, error) { - f.sourcePathRemaps = f.sourcePathRemaps[:0] - for key := range f.requiredTypes { - delete(f.requiredTypes, key) + imageTypeIndex *imageTypeIndex, + typesFilter fullNameFilter, + optionsFilter fullNameFilter, +) (bool, error) { + fmt.Println("---", fileDescriptor.GetName(), "---") + defer fmt.Println("------") + packageName := protoreflect.FullName(fileDescriptor.GetPackage()) + if packageName != "" { + // Check if filtered by the package name. + isIncluded, isExplicit := typesFilter.filter(packageName) + if !isIncluded && isExplicit { + // The package is excluded. + return false, nil + } } - if f.requiredTypes == nil { - f.requiredTypes = make(map[protoreflect.FullName]struct{}) + + fileImports := make(map[string]struct{}) + builder := &sourcePathsBuilder{ + filePath: fileDescriptor.GetName(), + imageTypeIndex: imageTypeIndex, + typesFilter: typesFilter, + optionsFilter: optionsFilter, + fileImports: fileImports, } - // Check file options. - if options := fileDescriptor.GetOptions(); options != nil { - optionsPath := []int32{fileOptionsTag} - if err := f.options(options, optionsPath); err != nil { - return nil, err - } + sourcePath := make(protoreflect.SourcePath, 0, 8) + + // Walk the file descriptor. + if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, builder.addRemapsForDescriptor); err != nil { + return false, err } - // Walk the file descriptor, collecting required types and marking options for deletion. - if err := walk.DescriptorProtosWithPath(fileDescriptor, f.visit); err != nil { - return nil, err + if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, builder.addRemapsForEnum); err != nil { + return false, err } - if len(f.sourcePathRemaps) == 0 { - return fileDescriptor, nil // No changes required. + if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileServicesTag), fileDescriptor.Service, builder.addRemapsForService); err != nil { + return false, err } - - // Recursively apply the source path remaps. - - // Convert the required types to imports. - fileImports := make(map[string]struct{}, len(fileDescriptor.Dependency)) - for requiredType := range f.requiredTypes { - descriptor, err := f.resolver.FindDescriptorByName(requiredType) - if err != nil { - return nil, fmt.Errorf("couldn't find type %s: %w", requiredType, err) - } - importPath := descriptor.ParentFile().Path() - if _, ok := fileImports[importPath]; ok || importPath == filePath { - continue - } - fileImports[importPath] = struct{}{} + if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, builder.addRemapsForField); err != nil { + return false, err } - // Fix imports. + if err := builder.addRemapsForOptions(sourcePathsRemap, append(sourcePath, fileOptionsTag), fileDescriptor.Options); err != nil { + return false, err + } + + // Fix the imports to remove any that are no longer used. + // TODO: handle unused dependencies, and keep them? + fmt.Println("fileImports", builder.fileImports) if len(fileImports) != len(fileDescriptor.Dependency) { indexTo := int32(0) dependencyPath := []int32{fileDependencyTag} @@ -179,11 +195,11 @@ func (f *protoOptionsFilter) fileDescriptor( if _, ok := fileImports[dependency]; ok { dependencyChanges[indexFrom] = indexTo if indexTo != int32(indexFrom) { - f.sourcePathRemaps.markMoved(path, indexTo) + sourcePathsRemap.markMoved(path, indexTo) } indexTo++ } else { - f.sourcePathRemaps.markDeleted(path) + sourcePathsRemap.markDeleted(path) dependencyChanges[indexFrom] = -1 } } @@ -192,9 +208,9 @@ func (f *protoOptionsFilter) fileDescriptor( path := append(publicDependencyPath, int32(indexFrom)) indexTo := dependencyChanges[publicDependency] if indexTo == -1 { - f.sourcePathRemaps.markDeleted(path) + sourcePathsRemap.markDeleted(path) } else if indexTo != int32(indexFrom) { - f.sourcePathRemaps.markMoved(path, indexTo) + sourcePathsRemap.markMoved(path, indexTo) } } weakDependencyPath := []int32{fileWeakDependencyTag} @@ -202,49 +218,403 @@ func (f *protoOptionsFilter) fileDescriptor( path := append(weakDependencyPath, int32(indexFrom)) indexTo := dependencyChanges[weakDependency] if indexTo == -1 { - f.sourcePathRemaps.markDeleted(path) + sourcePathsRemap.markDeleted(path) } else if indexTo != int32(indexFrom) { - f.sourcePathRemaps.markMoved(path, indexTo) + sourcePathsRemap.markMoved(path, indexTo) } } } - // Remap the source code info. - fileDescriptorMessage := fileDescriptor.ProtoReflect() - newFileDescriptorMessage, err := remapMessageReflect(fileDescriptorMessage, f.sourcePathRemaps) + return true, nil +} + +func (b *sourcePathsBuilder) addRemapsForDescriptor( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + descriptor *descriptorpb.DescriptorProto, +) (bool, error) { + fullName := getFullName(parentName, descriptor) + isIncluded, isExplicit := b.typesFilter.filter(fullName) + if !isIncluded && isExplicit { + // The type is excluded. + return false, nil + } + //// If the message is only enclosing an included message remove the fields. + //if isIncluded { + // if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { + // return false, err + // } + // if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { + // return false, err + // } + // for index, extensionRange := range descriptor.GetExtensionRange() { + // fmt.Println("\textensionRange", index, extensionRange) + // extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) + // if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { + // return false, err + // } + // } + // if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { + // return false, err + // } + //} else { + // sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) + // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionsTag)) + // for index := range descriptor.GetExtensionRange() { + // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag)) + // } + // sourcePathsRemap.markDeleted(append(sourcePath, messageOptionsTag)) + //} + + // Walk the nested types. + hasNestedTypes, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.addRemapsForDescriptor) if err != nil { - return nil, err + return false, err } - newFileDescriptor, _ := newFileDescriptorMessage.Interface().(*descriptorpb.FileDescriptorProto) - if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { - newLocations := make([]*descriptorpb.SourceCodeInfo_Location, 0, len(locations)) - for _, location := range locations { - oldPath := location.Path - newPath, noComment := f.sourcePathRemaps.newPath(oldPath) - if newPath == nil { - continue - } - if !slices.Equal(oldPath, newPath) || noComment { - location = shallowClone(location) - location.Path = newPath + isIncluded = isIncluded || hasNestedTypes + + // Walk the enum types. + hasEnums, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.addRemapsForEnum) + if err != nil { + return false, err + } + isIncluded = isIncluded || hasEnums + + // Walk the oneof types. + hasOneofs, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.addRemapsForOneof) + if err != nil { + return false, err + } + isIncluded = isIncluded || hasOneofs + + // If the message is only enclosing an included message remove the fields. + if isIncluded { + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { + return false, err + } + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { + return false, err + } + for index, extensionRange := range descriptor.GetExtensionRange() { + fmt.Println("\textensionRange", index, extensionRange) + extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) + if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { + return false, err } - if noComment { - location.LeadingDetachedComments = nil - location.LeadingComments = nil - location.TrailingComments = nil + } + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { + return false, err + } + } else { + + return isIncluded, nil +} + +func (b *sourcePathsBuilder) addRemapsForEnum( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + enum *descriptorpb.EnumDescriptorProto, +) (bool, error) { + fullName := getFullName(parentName, enum) + if isIncluded, _ := b.typesFilter.filter(fullName); !isIncluded { + // The type is excluded, enum values cannot be excluded individually. + return false, nil + } + + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, enumOptionsTag), enum.GetOptions()); err != nil { + return false, err + } + + // Walk the enum values. + for index, enumValue := range enum.Value { + enumValuePath := append(sourcePath, enumValuesTag, int32(index)) + enumValueOptionsPath := append(enumValuePath, enumValueOptionsTag) + if err := b.addRemapsForOptions(sourcePathsRemap, enumValueOptionsPath, enumValue.GetOptions()); err != nil { + return false, err + } + } + return true, nil +} + +func (b *sourcePathsBuilder) addRemapsForOneof( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + oneof *descriptorpb.OneofDescriptorProto, +) (bool, error) { + fullName := getFullName(parentName, oneof) + if isIncluded, _ := b.typesFilter.filter(fullName); !isIncluded { + // The type is excluded, enum values cannot be excluded individually. + return false, nil + } + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, oneofOptionsTag), oneof.GetOptions()); err != nil { + return false, err + } + return true, nil +} + +func (b *sourcePathsBuilder) addRemapsForService( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + service *descriptorpb.ServiceDescriptorProto, +) (bool, error) { + fullName := getFullName(parentName, service) + isIncluded, isExplicit := b.typesFilter.filter(fullName) + if !isIncluded && isExplicit { + // The type is excluded. + return false, nil + } + if isIncluded { + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, serviceOptionsTag), service.GetOptions()); err != nil { + return false, err + } + } + // Walk the service methods. + hasMethods, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, serviceMethodsTag), service.Method, b.addRemapsForMethod) + if err != nil { + return false, err + } + return isIncluded || hasMethods, nil +} + +func (b *sourcePathsBuilder) addRemapsForMethod( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + method *descriptorpb.MethodDescriptorProto, +) (bool, error) { + fullName := getFullName(parentName, method) + if isIncluded, _ := b.typesFilter.filter(fullName); !isIncluded { + // The type is excluded. + return false, nil + } + inputName := protoreflect.FullName(strings.TrimPrefix(method.GetInputType(), ".")) + if isIncluded, _ := b.typesFilter.filter(inputName); !isIncluded { + // The input type is excluded. + return false, fmt.Errorf("input type %s of method %s is excluded", inputName, fullName) + } + b.addRequiredType(inputName) + outputName := protoreflect.FullName(strings.TrimPrefix(method.GetOutputType(), ".")) + if isIncluded, _ := b.typesFilter.filter(outputName); !isIncluded { + // The output type is excluded. + return false, fmt.Errorf("output type %s of method %s is excluded", outputName, fullName) + } + b.addRequiredType(outputName) + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, methodOptionsTag), method.GetOptions()); err != nil { + return false, err + } + return true, nil +} + +func (b *sourcePathsBuilder) addRemapsForField( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + field *descriptorpb.FieldDescriptorProto, +) (bool, error) { + if field.Extendee != nil { + // This is an extension field. + extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) + if isIncluded, _ := b.typesFilter.filter(extendeeName); !isIncluded { + return false, nil + } + b.addRequiredType(extendeeName) + } + switch field.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_ENUM, + descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, + descriptorpb.FieldDescriptorProto_TYPE_GROUP: + typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) + if isIncluded, _ := b.typesFilter.filter(typeName); !isIncluded { + return false, nil + } + b.addRequiredType(typeName) + case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, + descriptorpb.FieldDescriptorProto_TYPE_FLOAT, + descriptorpb.FieldDescriptorProto_TYPE_INT64, + descriptorpb.FieldDescriptorProto_TYPE_UINT64, + descriptorpb.FieldDescriptorProto_TYPE_INT32, + descriptorpb.FieldDescriptorProto_TYPE_FIXED64, + descriptorpb.FieldDescriptorProto_TYPE_FIXED32, + descriptorpb.FieldDescriptorProto_TYPE_BOOL, + descriptorpb.FieldDescriptorProto_TYPE_STRING, + descriptorpb.FieldDescriptorProto_TYPE_BYTES, + descriptorpb.FieldDescriptorProto_TYPE_UINT32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, + descriptorpb.FieldDescriptorProto_TYPE_SINT32, + descriptorpb.FieldDescriptorProto_TYPE_SINT64: + default: + return false, fmt.Errorf("unknown field type %d", field.GetType()) + } + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, fieldOptionsTag), field.GetOptions()); err != nil { + return false, err + } + return true, nil +} + +func (b *sourcePathsBuilder) addRemapsForOptions( + sourcePathsRemap *sourcePathsRemapTrie, + optionsPath protoreflect.SourcePath, + optionsMessage proto.Message, +) error { + if optionsMessage == nil { + return nil + } + options := optionsMessage.ProtoReflect() + numFieldsToKeep := 0 + options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + isIncluded, _ := b.optionsFilter.filter(fd.FullName()) + if !isIncluded { + // Remove this option. + fmt.Println("\tremove option", fd.FullName()) + optionPath := append(optionsPath, int32(fd.Number())) + sourcePathsRemap.markDeleted(optionPath) + return true + } + fmt.Println("\tkeep option", fd.FullName()) + numFieldsToKeep++ + if fd.IsExtension() { + // Add the extension type to the required types. + b.addRequiredType(fd.FullName()) + } + return true + }) + if numFieldsToKeep == 0 { + // No options to keep. + sourcePathsRemap.markDeleted(optionsPath) + } + return nil +} + +func (b *sourcePathsBuilder) addRequiredType(fullName protoreflect.FullName) { + file, ok := b.imageTypeIndex.TypeToFile[fullName] + if !ok { + panic(fmt.Sprintf("could not find file for %s", fullName)) + } + fmt.Println("\taddRequiredType", fullName, file.Path()) + if file.Path() != b.filePath { + // This is an imported type. + file := b.imageTypeIndex.TypeToFile[fullName] + b.fileImports[file.Path()] = struct{}{} + } +} + +func addRemapsForSlice[T any]( + sourcePathsRemap *sourcePathsRemapTrie, + parentName protoreflect.FullName, + sourcePath protoreflect.SourcePath, + list []T, + addRemapsForItem func(*sourcePathsRemapTrie, protoreflect.FullName, protoreflect.SourcePath, T) (bool, error), +) (bool, error) { + fromIndex, toIndex := int32(0), int32(0) + for int(fromIndex) < len(list) { + item := list[fromIndex] + sourcePath := append(sourcePath, fromIndex) + isIncluded, err := addRemapsForItem(sourcePathsRemap, parentName, sourcePath, item) + if err != nil { + return false, err + } + if isIncluded { + if fromIndex != toIndex { + sourcePathsRemap.markMoved(sourcePath, toIndex) } - newLocations = append(newLocations, location) + toIndex++ + } else { + sourcePathsRemap.markDeleted(sourcePath) } - newFileDescriptor.SourceCodeInfo = &descriptorpb.SourceCodeInfo{ - Location: newLocations, + fromIndex++ + } + if toIndex == 0 { + sourcePathsRemap.markDeleted(sourcePath) + } + return toIndex > 0, nil +} + +/* +type fileDescriptorWalker struct { + filePath string + imageTypeIndex *imageTypeIndex + typesFilter fullNameFilter + optionsFilter fullNameFilter + + // On walking record whether the type is inlcuded. + includes []bool + + sourcePathsRemap sourcePathsRemapTrie + fileImports map[string]struct{} +} + +func (f *fileDescriptorWalker) walkFile( + fileDescriptor *descriptorpb.FileDescriptorProto, + sourcePathsRemap sourcePathsRemapTrie, +) error { + prefix := fileDescriptor.GetPackage() + if prefix != "" { + prefix += "." + } + isIncluded, isExplicit := f.typesFilter.filter(protoreflect.FullName(prefix)) + if !isIncluded && isExpli + + sourcePath := make(protoreflect.SourcePath, 0, 16) + toIndex := 0 + for fromIndex := range fileDescriptor.MessageType { + sourcePath := append(sourcePath, fileMessagesTag, int32(fromIndex)) + isIncluded, err := f.walkMessage(prefix, sourcePath, fileDescriptor.MessageType[fromIndex]) + if err != nil { + return err + } + if isIncluded && fromIndex != toIndex { + f.sourcePathsRemap.markMoved(sourcePath, int32(toIndex)) + } else { + f.sourcePathsRemap.markDeleted(sourcePath) } } - return newFileDescriptor, nil } -func (f *protoOptionsFilter) visit(fullName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor proto.Message) error { +func (f *fileDescriptorWalker) walkMessage(prefix string, sourcePath protoreflect.SourcePath, descriptor *descriptorpb.DescriptorProto) (bool, error) { + +} + +func (f *fileDescriptorWalker) enter(fullName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor proto.Message) error { + var isIncluded bool + switch descriptor := descriptor.(type) { + case *descriptorpb.EnumValueDescriptorProto, *descriptorpb.OneofDescriptorProto: + // Added by their enclosing types. + isIncluded = f.includes[len(f.includes)-1] + case *descriptorpb.FieldDescriptorProto: + isIncluded = f.isFieldIncluded(descriptor) + default: + isIncluded = f.typesFilter.filter(fullName) + } + fmt.Println("ENTER", fullName, "included?", isIncluded) + if isIncluded { + // If a child is included, the parent must be included. + for index := range f.includes { + f.includes[index] = true + } + } + f.includes = append(f.includes, isIncluded) + return nil +} + +func (f *fileDescriptorWalker) exit(fullName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor proto.Message) error { + fmt.Println("EXIT", f.includes) + isIncluded := f.includes[len(f.includes)-1] + f.includes = f.includes[:len(f.includes)-1] + if !isIncluded { + // Mark the source path for deletion. + f.sourcePathsRemap.markDeleted(sourcePath) + fmt.Println("\tDELETE", fullName) + return nil + } + // If the type is included, walk the options. switch descriptor := descriptor.(type) { case *descriptorpb.FileDescriptorProto: // File options are handled at the top level, before walking the file. + // The FileDescriptorProto is not walked here. return nil case *descriptorpb.DescriptorProto: optionsPath := append(sourcePath, messageOptionsTag) @@ -279,7 +649,7 @@ func (f *protoOptionsFilter) visit(fullName protoreflect.FullName, sourcePath pr } } -func (f *protoOptionsFilter) options( +func (f *fileDescriptorWalker) options( optionsMessage proto.Message, optionsPath protoreflect.SourcePath, ) error { @@ -290,41 +660,54 @@ func (f *protoOptionsFilter) options( if !options.IsValid() { return nil // No options to strip. } - numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - if !f.optionsFilter(fd, v) { + if !f.optionsFilter.filter(fd.FullName()) { // Remove this option. + fmt.Println("\tremove option", fd.FullName()) optionPath := append(optionsPath, int32(fd.Number())) - f.sourcePathRemaps.markDeleted(optionPath) + f.sourcePathsRemap.markDeleted(optionPath) return true } + fmt.Println("\tkeep option", fd.FullName()) numFieldsToKeep++ if fd.IsExtension() { // Add the extension type to the required types. - f.requiredTypes[fd.FullName()] = struct{}{} + f.addRequiredType(fd.FullName()) } return true }) if numFieldsToKeep == 0 { - // No options to keep. - f.sourcePathRemaps.markDeleted(optionsPath) + f.sourcePathsRemap.markDeleted(optionsPath) // No options to keep. } return nil } -func (f *protoOptionsFilter) addFieldType(field *descriptorpb.FieldDescriptorProto) error { +func (f *fileDescriptorWalker) isFieldIncluded(field *descriptorpb.FieldDescriptorProto) bool { + isIncluded := f.includes[len(f.includes)-1] + if field.Extendee != nil { + // This is an extension field. + extendee := strings.TrimPrefix(field.GetExtendee(), ".") + isIncluded = isIncluded && f.typesFilter.filter(protoreflect.FullName(extendee)) + } + typeName := strings.TrimPrefix(field.GetTypeName(), ".") + isIncluded = isIncluded && f.typesFilter.filter(protoreflect.FullName(typeName)) + fmt.Println("\tisFieldIncluded", field.GetName(), typeName, isIncluded) + return isIncluded +} + +func (f *fileDescriptorWalker) addFieldType(field *descriptorpb.FieldDescriptorProto) error { if field.Extendee != nil { // This is an extension field. extendee := strings.TrimPrefix(field.GetExtendee(), ".") - f.requiredTypes[protoreflect.FullName(extendee)] = struct{}{} + f.addRequiredType(protoreflect.FullName(extendee)) } switch field.GetType() { case descriptorpb.FieldDescriptorProto_TYPE_ENUM, descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, descriptorpb.FieldDescriptorProto_TYPE_GROUP: typeName := strings.TrimPrefix(field.GetTypeName(), ".") - f.requiredTypes[protoreflect.FullName(typeName)] = struct{}{} + f.addRequiredType(protoreflect.FullName(typeName)) case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, descriptorpb.FieldDescriptorProto_TYPE_FLOAT, descriptorpb.FieldDescriptorProto_TYPE_INT64, @@ -347,6 +730,56 @@ func (f *protoOptionsFilter) addFieldType(field *descriptorpb.FieldDescriptorPro return nil } +func (f *fileDescriptorWalker) addRequiredType(fullName protoreflect.FullName) { + file := f.imageTypeIndex.TypeToFile[fullName] + fmt.Println("\taddRequiredType", fullName, file.Path()) + if file.Path() != f.filePath { + // This is an imported type. + file := f.imageTypeIndex.TypeToFile[fullName] + f.fileImports[file.Path()] = struct{}{} + } +}*/ + +func remapFileDescriptor( + fileDescriptor *descriptorpb.FileDescriptorProto, + sourcePathRemaps sourcePathsRemapTrie, +) (*descriptorpb.FileDescriptorProto, error) { + fileDescriptorMessage := fileDescriptor.ProtoReflect() + newFileDescriptorMessage, err := remapMessageReflect(fileDescriptorMessage, sourcePathRemaps) + if err != nil { + return nil, err + } + newFileDescriptor, ok := newFileDescriptorMessage.Interface().(*descriptorpb.FileDescriptorProto) + if !ok { + return nil, syserror.Newf("unexpected type %T", newFileDescriptorMessage.Interface()) + } + // Remap the source code info. + if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { + newLocations := make([]*descriptorpb.SourceCodeInfo_Location, 0, len(locations)) + for _, location := range locations { + oldPath := location.Path + newPath, noComment := sourcePathRemaps.newPath(oldPath) + if newPath == nil { + continue + } + if !slices.Equal(oldPath, newPath) || noComment { + location = shallowClone(location) + location.Path = newPath + } + if noComment { + location.LeadingDetachedComments = nil + location.LeadingComments = nil + location.TrailingComments = nil + } + newLocations = append(newLocations, location) + } + newFileDescriptor.SourceCodeInfo = &descriptorpb.SourceCodeInfo{ + Location: newLocations, + } + } + return newFileDescriptor, nil +} + func remapMessageReflect( message protoreflect.Message, sourcePathRemaps sourcePathsRemapTrie, @@ -431,6 +864,11 @@ func remapListReflect( if fromIndex != int(remapNode.oldIndex) || toIndex != int(remapNode.newIndex) { return fmt.Errorf("unexpected list move %d to %d, expected %d to %d", remapNode.oldIndex, remapNode.newIndex, fromIndex, toIndex) } + //if toIndex != int(remapNode.newIndex) { + // // Mutate the remap node to reflect the actual index. + // // TODO: this is a hack. + // remapNode.newIndex = int32(toIndex) + //} // If no children, the value is unchanged. if len(remapNode.children) > 0 { // Must be a list of messages to have children. @@ -491,3 +929,11 @@ func shallowCloneReflect(src protoreflect.Message) protoreflect.Message { }) return dst } + +func getFullName(parentName protoreflect.FullName, message interface{ GetName() string }) protoreflect.FullName { + fullName := protoreflect.FullName(message.GetName()) + if parentName == "" { + return fullName + } + return parentName + "." + fullName +} diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go b/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go index 4805269d82..45b64c5e35 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go +++ b/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go @@ -47,75 +47,81 @@ func TestExcludeOptions(t *testing.T) { t.Run("ExcludeMessage", func(t *testing.T) { t.Parallel() testExcludeOptions( - t, "testdata/excludeoptions", - "message_foo", "message_bar", "message_baz", + t, "testdata/excludeoptions", WithExcludeOptions( + "message_foo", "message_bar", "message_baz", + ), ) }) t.Run("ExcludeFoo", func(t *testing.T) { t.Parallel() testExcludeOptions( - t, "testdata/excludeoptions", - "message_foo", - "field_foo", - "oneof_foo", - "enum_foo", - "enum_value_foo", - "service_foo", - "method_foo", - "UsedOption.file_foo", + t, "testdata/excludeoptions", WithExcludeOptions( + "message_foo", + "field_foo", + "oneof_foo", + "enum_foo", + "enum_value_foo", + "service_foo", + "method_foo", + "UsedOption.file_foo", + ), ) }) t.Run("OnlyFile", func(t *testing.T) { t.Parallel() testExcludeOptions( - t, "testdata/excludeoptions", - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "oneof_foo", "oneof_bar", "oneof_baz", - "enum_foo", "enum_bar", "enum_baz", - "enum_value_foo", "enum_value_bar", "enum_value_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", + t, "testdata/excludeoptions", WithExcludeOptions( + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + ), ) }) t.Run("OnlyOneOf", func(t *testing.T) { t.Parallel() testExcludeOptions( - t, "testdata/excludeoptions", - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "enum_foo", "enum_bar", "enum_baz", - "enum_value_foo", "enum_value_bar", "enum_value_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + t, "testdata/excludeoptions", WithExcludeOptions( + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + ), ) }) t.Run("OnlyEnumValue", func(t *testing.T) { t.Parallel() testExcludeOptions( - t, "testdata/excludeoptions", - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "oneof_foo", "oneof_bar", "oneof_baz", - "enum_foo", "enum_bar", "enum_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + t, "testdata/excludeoptions", WithExcludeOptions( + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + ), ) }) t.Run("ExcludeAll", func(t *testing.T) { t.Parallel() testExcludeOptions( - t, "testdata/excludeoptions", - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "oneof_foo", "oneof_bar", "oneof_baz", - "enum_foo", "enum_bar", "enum_baz", - "enum_value_foo", "enum_value_bar", "enum_value_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + t, "testdata/excludeoptions", WithExcludeOptions( + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", + ), ) }) } @@ -155,25 +161,63 @@ func TestExcludeOptionImports(t *testing.T) { }) t.Run("ExcludeFoo", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, "message_foo") + testExcludeOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo")) }) t.Run("ExcludeFooBar", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, "message_foo", "message_bar") + testExcludeOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo", "message_bar")) }) t.Run("ExcludeBar", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, "message_bar") + testExcludeOptionsForImage(t, bucket, image, WithExcludeOptions("message_bar")) }) } -func testExcludeOptions(t *testing.T, testdataDir string, options ...string) { +func TestFilterTypes(t *testing.T) { + t.Parallel() + + testdataDir := "testdata/nesting" + bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) + require.NoError(t, err) + testModuleData := []bufmoduletesting.ModuleData{ + { + Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("a.proto")), + }, + { + Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("options.proto")), + NotTargeted: true, + }, + } + moduleSet, err := bufmoduletesting.NewModuleSet(testModuleData...) + require.NoError(t, err) + + // Safe to filter the image concurrently as its not being modified. + image, err := bufimage.BuildImage( + context.Background(), + slogtestext.NewLogger(t), + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), + bufimage.WithExcludeSourceCodeInfo(), + ) + require.NoError(t, err) + + t.Run("ExcludeBar", func(t *testing.T) { + t.Parallel() + testExcludeOptionsForImage(t, bucket, image, WithExcludeTypes("pkg.Bar")) + }) + t.Run("IncludeBar", func(t *testing.T) { + t.Parallel() + testExcludeOptionsForImage(t, bucket, image, WithIncludeTypes("pkg.Bar")) + }) + +} + +func testExcludeOptions(t *testing.T, testdataDir string, options ...ImageFilterOption) { bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) require.NoError(t, err) testExcludeOptionsForModuleData(t, bucket, nil, options...) } -func testExcludeOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucket, moduleData []bufmoduletesting.ModuleData, options ...string) { +func testExcludeOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucket, moduleData []bufmoduletesting.ModuleData, options ...ImageFilterOption) { ctx := context.Background() if len(moduleData) == 0 { moduleData = append(moduleData, bufmoduletesting.ModuleData{ @@ -194,9 +238,9 @@ func testExcludeOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucke testExcludeOptionsForImage(t, bucket, image, options...) } -func testExcludeOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, image bufimage.Image, options ...string) { +func testExcludeOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, image bufimage.Image, options ...ImageFilterOption) { ctx := context.Background() - filteredImage, err := ExcludeOptions(image, options...) + filteredImage, err := FilterImage(image, options...) require.NoError(t, err) files, err := protodesc.NewFiles(&descriptorpb.FileDescriptorSet{ diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 7840c50f27..0eb7c33a30 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -25,6 +25,8 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) +// file -> DAG types. + // imageIndex holds an index that allows for easily navigating a descriptor // hierarchy and its relationships. type imageIndex struct { diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go new file mode 100644 index 0000000000..4d2b1f81fa --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/image_types.go @@ -0,0 +1,305 @@ +// Copyright 2020-2025 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimageutil + +import ( + "fmt" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/protocompile/walk" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" +) + +type typeLink int + +const ( + typeLinkNone typeLink = iota // 0 + typeLinkChild // 1 + typeLinkDepends // 2 + typeLinkOption // 3 +) + +type imageTypeIndex struct { + // TypeSet maps fully qualified type names to their children. + TypeSet map[protoreflect.FullName]map[protoreflect.FullName]typeLink + // TypeToFile maps fully qualified type names to their image file. + TypeToFile map[protoreflect.FullName]bufimage.ImageFile +} + +func newImageTypeIndex(image bufimage.Image) (*imageTypeIndex, error) { + index := &imageTypeIndex{ + TypeSet: make(map[protoreflect.FullName]map[protoreflect.FullName]typeLink), + TypeToFile: make(map[protoreflect.FullName]bufimage.ImageFile), + } + for _, file := range image.Files() { + if err := index.addFile(file); err != nil { + return nil, err + } + } + return index, nil +} + +func (i *imageTypeIndex) addFile(file bufimage.ImageFile) error { + fileDescriptor := file.FileDescriptorProto() + packageName := protoreflect.FullName(fileDescriptor.GetPackage()) + // Add all parent packages to point to this package. + for packageName := packageName; packageName != ""; { + sep := strings.LastIndex(string(packageName), ".") + if sep == -1 { + break + } + parentPackageName := protoreflect.FullName(packageName[:sep]) + i.linkType(parentPackageName, packageName, typeLinkChild) + packageName = parentPackageName + } + + stack := make([]protoreflect.FullName, 0, 10) + stack = append(stack, packageName) + i.addType(stack[0]) + i.addOptionTypes(stack[0], fileDescriptor.GetOptions()) + enter := func(fullName protoreflect.FullName, descriptor proto.Message) error { + fmt.Println("stack", stack, "->", fullName) + parentFullName := stack[len(stack)-1] + switch descriptor := descriptor.(type) { + case *descriptorpb.DescriptorProto: + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.FieldDescriptorProto: + i.addFieldType(parentFullName, descriptor) + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.OneofDescriptorProto: + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.EnumDescriptorProto: + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.EnumValueDescriptorProto: + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.ServiceDescriptorProto: + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.MethodDescriptorProto: + inputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetInputType(), ".")) + outputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetOutputType(), ".")) + i.linkType(parentFullName, inputName, typeLinkDepends) + i.linkType(parentFullName, outputName, typeLinkDepends) + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + case *descriptorpb.DescriptorProto_ExtensionRange: + i.addOptionTypes(parentFullName, descriptor.GetOptions()) + default: + return fmt.Errorf("unexpected message type %T", descriptor) + } + i.TypeToFile[fullName] = file + if isDescriptorType(descriptor) { + i.linkType(parentFullName, fullName, typeLinkChild) + i.addType(fullName) + stack = append(stack, fullName) + } + return nil + } + exit := func(fullName protoreflect.FullName, descriptor proto.Message) error { + if isDescriptorType(descriptor) { + stack = stack[:len(stack)-1] + } + fmt.Println("exit ", stack, "->", fullName) + return nil + } + if err := walk.DescriptorProtosEnterAndExit(fileDescriptor, enter, exit); err != nil { + return err + } + return nil +} + +func (i *imageTypeIndex) addType(fullName protoreflect.FullName) { + if _, ok := i.TypeSet[fullName]; !ok { + i.TypeSet[fullName] = nil + } +} + +func (i *imageTypeIndex) linkType(parentFullName protoreflect.FullName, fullName protoreflect.FullName, link typeLink) { + if typeSet := i.TypeSet[parentFullName]; typeSet != nil { + typeSet[fullName] = link + } else { + i.TypeSet[parentFullName] = map[protoreflect.FullName]typeLink{fullName: link} + } +} + +func (i *imageTypeIndex) addFieldType(parentFullName protoreflect.FullName, fieldDescriptor *descriptorpb.FieldDescriptorProto) { + if extendee := fieldDescriptor.GetExtendee(); extendee != "" { + // This is an extension field. + extendeeFullName := protoreflect.FullName(strings.TrimPrefix(extendee, ".")) + i.linkType(parentFullName, extendeeFullName, typeLinkDepends) + } + // Add the field type. + switch fieldDescriptor.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_ENUM, + descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, + descriptorpb.FieldDescriptorProto_TYPE_GROUP: + // Add links to the type of the field. + typeFullName := protoreflect.FullName( + strings.TrimPrefix(fieldDescriptor.GetTypeName(), "."), + ) + i.linkType(parentFullName, typeFullName, typeLinkDepends) + } +} + +func (i *imageTypeIndex) addOptionTypes(parentFullName protoreflect.FullName, optionsMessage proto.Message) { + if optionsMessage == nil { + return + } + options := optionsMessage.ProtoReflect() + if !options.IsValid() { + return + } + options.Range(func(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) bool { + if fieldDescriptor.IsExtension() { + i.linkType(parentFullName, fieldDescriptor.FullName(), typeLinkOption) + } + return true + }) +} + +type fullNameFilter struct { + include map[protoreflect.FullName]struct{} + exclude map[protoreflect.FullName]struct{} +} + +func (f *fullNameFilter) filter(fullName protoreflect.FullName) (isIncluded bool, isExplicit bool) { + if f.exclude != nil { + if _, ok := f.exclude[fullName]; ok { + return false, true + } + } + if f.include != nil { + if _, ok := f.include[fullName]; ok { + return true, true + } + return false, false + } + return true, false +} + +func createTypeFilter(index *imageTypeIndex, options *imageFilterOptions) (fullNameFilter, error) { + var filter fullNameFilter + var excludeList []protoreflect.FullName + for excludeType := range options.excludeTypes { + excludeType := protoreflect.FullName(excludeType) + if _, ok := index.TypeSet[excludeType]; !ok { + return filter, fmt.Errorf("filtering by excluded type %q: %w", excludeType, ErrImageFilterTypeNotFound) + } + file := index.TypeToFile[excludeType] + if file.IsImport() && !options.allowImportedTypes { + return filter, fmt.Errorf("filtering by excluded type %q: %w", excludeType, ErrImageFilterTypeIsImport) + } + excludeList = append(excludeList, excludeType) + } + if len(excludeList) > 0 { + filter.exclude = make(map[protoreflect.FullName]struct{}) + } + for len(excludeList) > 0 { + excludeType := excludeList[len(excludeList)-1] + excludeList = excludeList[:len(excludeList)-1] + if _, ok := filter.exclude[excludeType]; ok { + continue + } + for childType, childLink := range index.TypeSet[excludeType] { + if childLink == typeLinkChild { + excludeList = append(excludeList, childType) + } + } + filter.exclude[excludeType] = struct{}{} + } + + var includeList []protoreflect.FullName + for includeType := range options.includeTypes { + includeType := protoreflect.FullName(includeType) + if _, ok := index.TypeSet[includeType]; !ok { + return filter, fmt.Errorf("filtering by included type %q: %w", includeType, ErrImageFilterTypeNotFound) + } + file := index.TypeToFile[includeType] + if file.IsImport() && !options.allowImportedTypes { + return filter, fmt.Errorf("filtering by included type %q: %w", includeType, ErrImageFilterTypeIsImport) + } + if _, ok := filter.exclude[includeType]; ok { + continue // Skip already excluded. + } + includeList = append(includeList, includeType) + } + if len(includeList) > 0 { + filter.include = make(map[protoreflect.FullName]struct{}) + } + for len(includeList) > 0 { + includeType := includeList[len(includeList)-1] + includeList = includeList[:len(includeList)-1] + if _, ok := filter.include[includeType]; ok { + continue + } + for childType, childLink := range index.TypeSet[includeType] { + if _, ok := filter.exclude[includeType]; ok { + continue // Skip already excluded. + } + switch childLink { + case typeLinkChild: + includeList = append(includeList, childType) + case typeLinkDepends: + includeList = append(includeList, childType) + case typeLinkOption: + if options.includeKnownExtensions || (options.includeCustomOptions && isOptionsTypeName(string(childType))) { + includeList = append(includeList, childType) + } + } + } + filter.include[includeType] = struct{}{} + } + return filter, nil +} + +func createOptionsFilter(index *imageTypeIndex, options *imageFilterOptions) (fullNameFilter, error) { + var filter fullNameFilter + for includeOption := range options.includeOptions { + includeOption := protoreflect.FullName(includeOption) + if _, ok := index.TypeSet[includeOption]; !ok { + return filter, fmt.Errorf("filtering by included option %q: %w", includeOption, ErrImageFilterTypeNotFound) + } + // TODO: check for imported type filter? + if filter.include == nil { + filter.include = make(map[protoreflect.FullName]struct{}) + } + filter.include[includeOption] = struct{}{} + } + for excludeOption := range options.excludeOptions { + excludeOption := protoreflect.FullName(excludeOption) + if _, ok := index.TypeSet[excludeOption]; !ok { + return filter, fmt.Errorf("filtering by excluded option %q: %w", excludeOption, ErrImageFilterTypeNotFound) + } + // TODO: check for imported type filter? + if filter.exclude == nil { + filter.exclude = make(map[protoreflect.FullName]struct{}) + } + filter.exclude[excludeOption] = struct{}{} + } + return filter, nil +} + +func isDescriptorType(descriptor proto.Message) bool { + switch descriptor := descriptor.(type) { + case *descriptorpb.EnumValueDescriptorProto, *descriptorpb.OneofDescriptorProto: + // Added by their enclosing types. + return false + case *descriptorpb.FieldDescriptorProto: + return descriptor.Extendee != nil + default: + return true + } +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto new file mode 100644 index 0000000000..7cffe7ce41 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; +package pkg; + +message A { + string a = 1; + message NestedA { + message NestedNestedA { + string nested_nested_x = 1; + } + string nested_x = 1; + } + NestedA nested_a = 1; +} + +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} + +message B { + enum NestedBEnum { + NESTED_B_ENUM_X = 0; + NESTED_B_ENUM_Y = 1; + } + FooEnum foo_enum = 1; + Foo.NestedA nested_a = 2; +} + +message Baz { + Bar.NestedBarEnum nested_bar_enum = 1; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/b.proto b/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/b.proto new file mode 100644 index 0000000000..e69de29bb2 diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar new file mode 100644 index 0000000000..d951fbac3f --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar @@ -0,0 +1,22 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Baz { +} +message Foo { + string x = 1; + NestedFoo nested_foo = 2; + message NestedButNotUsed { + string nested_but_not_used = 1; + } + message NestedFoo { + string nested_x = 1; + message NestedNestedFoo { + string nested_nested_x = 1; + } + } +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar new file mode 100644 index 0000000000..72a9e3683f --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar @@ -0,0 +1,23 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Bar { + FooEnum foo_enum = 1; + Foo.NestedFoo nested_foo = 2; + enum NestedBarEnum { + NESTED_BAR_ENUM_X = 0; + NESTED_BAR_ENUM_Y = 1; + } +} +message Foo { + message NestedFoo { + string nested_x = 1; + message NestedNestedFoo { + string nested_nested_x = 1; + } + } +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} From 4935430f56c2a2863fd075f71172356538feb6cb Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 24 Feb 2025 17:58:27 +0100 Subject: [PATCH 06/80] Refactor for include and exclude of types --- .../bufimage/bufimageutil/bufimageutil.go | 712 ------------------ .../{exclude_options.go => image_filter.go} | 393 ++-------- ...e_options_test.go => image_filter_test.go} | 36 +- .../bufimage/bufimageutil/image_index.go | 75 +- .../bufimage/bufimageutil/image_types.go | 629 ++++++++++------ 5 files changed, 545 insertions(+), 1300 deletions(-) rename private/bufpkg/bufimage/bufimageutil/{exclude_options.go => image_filter.go} (63%) rename private/bufpkg/bufimage/bufimageutil/{exclude_options_test.go => image_filter_test.go} (87%) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index fe36c552cb..6c88624e43 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -232,198 +232,6 @@ func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.I // allows for customizing the behavior with options. func ImageFilteredByTypesWithOptions(image bufimage.Image, types []string, opts ...ImageFilterOption) (bufimage.Image, error) { return FilterImage(image, append(opts, WithIncludeTypes(types...))...) - /*options := newImageFilterOptions() - for _, o := range opts { - o(options) - } - - imageIndex, err := newImageIndexForImage(image, options) - if err != nil { - return nil, err - } - // Check types exist - startingDescriptors := make([]namedDescriptor, 0, len(types)) - var startingPackages []*protoPackage - for _, typeName := range types { - // TODO: consider supporting a glob syntax of some kind, to do more advanced pattern - // matching, such as ability to get a package AND all of its sub-packages. - startingDescriptor, ok := imageIndex.ByName[typeName] - if ok { - // It's a type name - typeInfo := imageIndex.ByDescriptor[startingDescriptor] - if image.GetFile(typeInfo.file).IsImport() && !options.allowImportedTypes { - return nil, fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) - } - startingDescriptors = append(startingDescriptors, startingDescriptor) - continue - } - // It could be a package name - pkg, ok := imageIndex.Packages[typeName] - if !ok { - // but it's not... - return nil, fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeNotFound) - } - if !options.allowImportedTypes { - // if package includes only imported files, then reject - onlyImported := true - for _, file := range pkg.files { - if !file.IsImport() { - onlyImported = false - break - } - } - if onlyImported { - return nil, fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) - } - } - startingPackages = append(startingPackages, pkg) - } - // Find all types to include in filtered image. - closure := newTransitiveClosure() - for _, startingPackage := range startingPackages { - if err := closure.addPackage(startingPackage, imageIndex, options); err != nil { - return nil, err - } - } - for _, startingDescriptor := range startingDescriptors { - if err := closure.addElement(startingDescriptor, "", false, imageIndex, options); err != nil { - return nil, err - } - } - // After all types are added, add their known extensions - if err := closure.addExtensions(imageIndex, options); err != nil { - return nil, err - } - // Create a new image with only the required descriptors. - var includedFiles []bufimage.ImageFile - for _, imageFile := range image.Files() { - _, ok := closure.files[imageFile.Path()] - if !ok { - continue - } - includedFiles = append(includedFiles, imageFile) - imageFileDescriptor := imageFile.FileDescriptorProto() - - importsRequired := closure.imports[imageFile.Path()] - // If the file has source code info, we need to remap paths to correctly - // update this info for the elements retained after filtering. - var sourcePathRemapper *sourcePathsRemapTrie - if len(imageFileDescriptor.SourceCodeInfo.GetLocation()) > 0 { - sourcePathRemapper = &sourcePathsRemapTrie{} - } - // We track the source path as we go through the model, so that we can - // mark paths as moved or deleted. Whenever an element is deleted, any - // subsequent elements of the same type in the same scope have are "moved", - // because their index is shifted down. - basePath := make([]int32, 1, 16) - basePath[0] = fileDependencyTag - // While employing - // https://github.com/golang/go/wiki/SliceTricks#filter-in-place, - // also keep a record of which index moved where, so we can fixup - // the file's WeakDependency field. - indexFromTo := make(map[int32]int32) - indexTo := 0 - // Only handle imports and dependencies if there are any. - for indexFrom, importPath := range imageFileDescriptor.GetDependency() { - path := append(basePath, int32(indexFrom)) - // We check if the import path exists among required imports. If yes, we - // move and then delete from required imports as we go. - if importsRequired != nil && importsRequired.index(importPath) != -1 { - sourcePathRemapper.markMoved(path, int32(indexTo)) - indexFromTo[int32(indexFrom)] = int32(indexTo) - imageFileDescriptor.Dependency[indexTo] = importPath - indexTo++ - // delete them as we go, so we know which ones weren't in the list - importsRequired.delete(importPath) - } else { - // Path did not exist in required imports, we mark as deleted. - sourcePathRemapper.markDeleted(path) - } - } - imageFileDescriptor.Dependency = imageFileDescriptor.Dependency[:indexTo] - - // Add any other imports (which may not have been in the list because - // they were picked up via a public import). The filtered files will not - // use public imports. - // The imports are added in the order they are encountered when importing - // to maintain a deterministic ordering. - if importsRequired != nil { - imageFileDescriptor.Dependency = append(imageFileDescriptor.Dependency, importsRequired.keys()...) - } - - imageFileDescriptor.PublicDependency = nil - sourcePathRemapper.markDeleted([]int32{filePublicDependencyTag}) - - basePath = basePath[:1] - basePath[0] = fileWeakDependencyTag - i := 0 - for _, indexFrom := range imageFileDescriptor.WeakDependency { - path := append(basePath, indexFrom) - if indexTo, ok := indexFromTo[indexFrom]; ok { - sourcePathRemapper.markMoved(path, indexTo) - imageFileDescriptor.WeakDependency[i] = indexTo - i++ - } else { - sourcePathRemapper.markDeleted(path) - } - } - imageFileDescriptor.WeakDependency = imageFileDescriptor.WeakDependency[:i] - - if _, ok := closure.completeFiles[imageFile.Path()]; !ok { - // if not keeping entire file, filter contents now - basePath = basePath[:0] - imageFileDescriptor.MessageType = trimMessageDescriptors(imageFileDescriptor.MessageType, closure.elements, sourcePathRemapper, append(basePath, fileMessagesTag)) - imageFileDescriptor.EnumType = trimSlice(imageFileDescriptor.EnumType, closure.elements, sourcePathRemapper, append(basePath, fileEnumsTag)) - // TODO: We could end up removing all extensions from a particular extend block - // but we then don't mark that extend block's source code info for deletion. This - // is because extend blocks don't have distinct paths -- we have to actually look - // at the span information to determine which extensions correspond to which blocks - // to decide which blocks to remove. That is possible, but non-trivial, and it's - // unclear if the "juice is worth the squeeze", so we leave it. The best we do is - // to remove comments for extend blocks when there are NO extensions. - extsPath := append(basePath, fileExtensionsTag) - imageFileDescriptor.Extension = trimSlice(imageFileDescriptor.Extension, closure.elements, sourcePathRemapper, extsPath) - if len(imageFileDescriptor.Extension) == 0 { - sourcePathRemapper.markDeleted(extsPath) - } - svcsPath := append(basePath, fileServicesTag) - // We must iterate through the services *before* we trim the slice. That way the - // index we see is for the "old path", which we need to know to mark elements as - // moved or deleted with the sourcePathRemapper. - for index, serviceDescriptor := range imageFileDescriptor.Service { - if _, ok := closure.elements[serviceDescriptor]; !ok { - continue - } - methodPath := append(svcsPath, int32(index), serviceMethodsTag) - serviceDescriptor.Method = trimSlice(serviceDescriptor.Method, closure.elements, sourcePathRemapper, methodPath) - } - imageFileDescriptor.Service = trimSlice(imageFileDescriptor.Service, closure.elements, sourcePathRemapper, svcsPath) - } - - if len(imageFileDescriptor.SourceCodeInfo.GetLocation()) > 0 { - // Now the sourcePathRemapper has been fully populated for all of the deletions - // and moves above. So we can use it to reconstruct the source code info slice - // of locations. - i := 0 - for _, location := range imageFileDescriptor.SourceCodeInfo.Location { - // This function returns newPath==nil if the element at the given path - // was marked for deletion (so this location should be omitted). - newPath, noComment := sourcePathRemapper.newPath(location.Path) - if newPath != nil { - imageFileDescriptor.SourceCodeInfo.Location[i] = location - location.Path = newPath - if noComment { - location.LeadingDetachedComments = nil - location.LeadingComments = nil - location.TrailingComments = nil - } - i++ - } - } - imageFileDescriptor.SourceCodeInfo.Location = imageFileDescriptor.SourceCodeInfo.Location[:i] - } - } - return bufimage.NewImage(includedFiles)*/ } // StripSourceRetentionOptions strips any options with a retention of "source" from @@ -441,526 +249,6 @@ func StripSourceRetentionOptions(image bufimage.Image) (bufimage.Image, error) { return bufimage.NewImage(updatedFiles) } -/* -// trimMessageDescriptors removes (nested) messages and nested enums from a slice -// of message descriptors if their type names are not found in the toKeep map. -func trimMessageDescriptors( - in []*descriptorpb.DescriptorProto, - toKeep map[namedDescriptor]closureInclusionMode, - sourcePathRemapper *sourcePathsRemapTrie, - pathSoFar []int32, -) []*descriptorpb.DescriptorProto { - // We must iterate through the messages *before* we trim the slice. That way the - // index we see is for the "old path", which we need to know to mark elements as - // moved or deleted with the sourcePathRemapper. - for index, messageDescriptor := range in { - path := append(pathSoFar, int32(index)) - mode, ok := toKeep[messageDescriptor] - if !ok { - continue - } - if mode == inclusionModeEnclosing { - // if this is just an enclosing element, we only care about it as a namespace for - // other types and don't care about the rest of its contents - messageDescriptor.Field = nil - messageDescriptor.OneofDecl = nil - messageDescriptor.ExtensionRange = nil - messageDescriptor.ReservedRange = nil - messageDescriptor.ReservedName = nil - sourcePathRemapper.markNoComment(path) - sourcePathRemapper.markDeleted(append(path, messageFieldsTag)) - sourcePathRemapper.markDeleted(append(path, messageOneofsTag)) - sourcePathRemapper.markDeleted(append(path, messageExtensionRangesTag)) - sourcePathRemapper.markDeleted(append(path, messageReservedRangesTag)) - sourcePathRemapper.markDeleted(append(path, messageReservedNamesTag)) - } - messageDescriptor.NestedType = trimMessageDescriptors(messageDescriptor.NestedType, toKeep, sourcePathRemapper, append(path, messageNestedMessagesTag)) - messageDescriptor.EnumType = trimSlice(messageDescriptor.EnumType, toKeep, sourcePathRemapper, append(path, messageEnumsTag)) - // TODO: We could end up removing all extensions from a particular extend block - // but we then don't mark that extend block's source code info for deletion. The - // best we do is to remove comments for extend blocks when there are NO extensions. - // See comment above for file extensions for more info. - extsPath := append(path, messageExtensionsTag) - messageDescriptor.Extension = trimSlice(messageDescriptor.Extension, toKeep, sourcePathRemapper, extsPath) - if len(messageDescriptor.Extension) == 0 { - sourcePathRemapper.markDeleted(extsPath) - } - } - return trimSlice(in, toKeep, sourcePathRemapper, pathSoFar) -} - -// trimSlice removes elements from a slice of descriptors if they are -// not present in the given map. -func trimSlice[T namedDescriptor]( - in []T, - toKeep map[namedDescriptor]closureInclusionMode, - sourcePathRemapper *sourcePathsRemapTrie, - pathSoFar []int32, -) []T { - i := 0 - for index, descriptor := range in { - path := append(pathSoFar, int32(index)) - if _, ok := toKeep[descriptor]; ok { - sourcePathRemapper.markMoved(path, int32(i)) - in[i] = descriptor - i++ - } else { - sourcePathRemapper.markDeleted(path) - } - } - return in[:i] -} -/* -// transitiveClosure accumulates the elements, files, and needed imports for a -// subset of an image. When an element is added to the closure, all of its -// dependencies are recursively added. -type transitiveClosure struct { - // The elements included in the transitive closure. - elements map[namedDescriptor]closureInclusionMode - // The set of files that contain all items in elements. - files map[string]struct{} - // Any files that are part of the closure in their entirety (due to an - // entire package being included). The above fields are used to filter the - // contents of files. But files named in this set will not be filtered. - completeFiles map[string]struct{} - // The ordered set of imports for each file. This allows for re-writing imports - // for files whose contents have been pruned. - imports map[string]*orderedImports -} - -type closureInclusionMode int - -const ( - // Element is included in closure because it is directly reachable from a root. - inclusionModeExplicit = closureInclusionMode(iota) - // Element is included in closure because it is a message or service that - // *contains* an explicitly included element but is not itself directly - // reachable. - inclusionModeEnclosing - // Element is included in closure because it is implied by the presence of a - // custom option. For example, a field element with a custom option implies - // the presence of google.protobuf.FieldOptions. An option type could instead be - // explicitly included if it is also directly reachable (i.e. some type in the - // graph explicitly refers to the option type). - inclusionModeImplicit -) - -func newTransitiveClosure() *transitiveClosure { - return &transitiveClosure{ - elements: map[namedDescriptor]closureInclusionMode{}, - files: map[string]struct{}{}, - completeFiles: map[string]struct{}{}, - imports: map[string]*orderedImports{}, - } -} - -func (t *transitiveClosure) addImport(fromPath, toPath string) { - if fromPath == toPath { - return // no need for a file to import itself - } - imps := t.imports[fromPath] - if imps == nil { - imps = newOrderedImports() - t.imports[fromPath] = imps - } - imps.add(toPath) -} - -func (t *transitiveClosure) addFile(file string, imageIndex *imageIndex, opts *imageFilterOptions) error { - if _, ok := t.files[file]; ok { - return nil // already added - } - t.files[file] = struct{}{} - return t.exploreCustomOptions(imageIndex.Files[file], file, imageIndex, opts) -} - -func (t *transitiveClosure) addPackage( - pkg *protoPackage, - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - for _, file := range pkg.files { - if err := t.addFile(file.Path(), imageIndex, opts); err != nil { - return err - } - t.completeFiles[file.Path()] = struct{}{} - } - for _, descriptor := range pkg.elements { - if err := t.addElement(descriptor, "", false, imageIndex, opts); err != nil { - return err - } - } - return nil -} - -func (t *transitiveClosure) addElement( - descriptor namedDescriptor, - referrerFile string, - impliedByCustomOption bool, - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - descriptorInfo := imageIndex.ByDescriptor[descriptor] - if err := t.addFile(descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - if referrerFile != "" { - t.addImport(referrerFile, descriptorInfo.file) - } - - if existingMode, ok := t.elements[descriptor]; ok && existingMode != inclusionModeEnclosing { - if existingMode == inclusionModeImplicit && !impliedByCustomOption { - // upgrade from implied to explicitly part of closure - t.elements[descriptor] = inclusionModeExplicit - } - return nil // already added this element - } - if impliedByCustomOption { - t.elements[descriptor] = inclusionModeImplicit - } else { - t.elements[descriptor] = inclusionModeExplicit - } - - // if this type is enclosed inside another, add enclosing types - if err := t.addEnclosing(descriptorInfo.parent, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - // add any custom options and their dependencies - if err := t.exploreCustomOptions(descriptor, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - - switch typedDescriptor := descriptor.(type) { - case *descriptorpb.DescriptorProto: - // Options and types for all fields - for _, field := range typedDescriptor.GetField() { - if err := t.addFieldType(field, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - if err := t.exploreCustomOptions(field, referrerFile, imageIndex, opts); err != nil { - return err - } - } - // Options for all oneofs in this message - for _, oneOfDescriptor := range typedDescriptor.GetOneofDecl() { - if err := t.exploreCustomOptions(oneOfDescriptor, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - } - // Options for all extension ranges in this message - for _, extRange := range typedDescriptor.GetExtensionRange() { - if err := t.exploreCustomOptions(extRange, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - } - - case *descriptorpb.EnumDescriptorProto: - for _, enumValue := range typedDescriptor.GetValue() { - if err := t.exploreCustomOptions(enumValue, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - } - - case *descriptorpb.ServiceDescriptorProto: - for _, method := range typedDescriptor.GetMethod() { - if err := t.addElement(method, "", false, imageIndex, opts); err != nil { - return err - } - } - - case *descriptorpb.MethodDescriptorProto: - inputName := strings.TrimPrefix(typedDescriptor.GetInputType(), ".") - inputDescriptor, ok := imageIndex.NameToDescriptor[inputName] - if !ok { - return fmt.Errorf("missing %q", inputName) - } - if err := t.addElement(inputDescriptor, descriptorInfo.file, false, imageIndex, opts); err != nil { - return err - } - - outputName := strings.TrimPrefix(typedDescriptor.GetOutputType(), ".") - outputDescriptor, ok := imageIndex.NameToDescriptor[outputName] - if !ok { - return fmt.Errorf("missing %q", outputName) - } - if err := t.addElement(outputDescriptor, descriptorInfo.file, false, imageIndex, opts); err != nil { - return err - } - - case *descriptorpb.FieldDescriptorProto: - // Regular fields are handled above in message descriptor case. - // We should only find our way here for extensions. - if typedDescriptor.Extendee == nil { - return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) - } - if typedDescriptor.GetExtendee() == "" { - return fmt.Errorf("expected extendee for field %q to not be empty", descriptorInfo.fullName) - } - extendeeName := strings.TrimPrefix(typedDescriptor.GetExtendee(), ".") - extendeeDescriptor, ok := imageIndex.NameToDescriptor[extendeeName] - if !ok { - return fmt.Errorf("missing %q", extendeeName) - } - if err := t.addElement(extendeeDescriptor, descriptorInfo.file, impliedByCustomOption, imageIndex, opts); err != nil { - return err - } - if err := t.addFieldType(typedDescriptor, descriptorInfo.file, imageIndex, opts); err != nil { - return err - } - - default: - return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) - } - - return nil -} - -func errorUnsupportedFilterType(descriptor namedDescriptor, fullName string) error { - var descriptorType string - switch d := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - descriptorType = "file" - case *descriptorpb.DescriptorProto: - descriptorType = "message" - case *descriptorpb.FieldDescriptorProto: - if d.Extendee != nil { - descriptorType = "extension field" - } else { - descriptorType = "non-extension field" - } - case *descriptorpb.OneofDescriptorProto: - descriptorType = "oneof" - case *descriptorpb.EnumDescriptorProto: - descriptorType = "enum" - case *descriptorpb.EnumValueDescriptorProto: - descriptorType = "enum value" - case *descriptorpb.ServiceDescriptorProto: - descriptorType = "service" - case *descriptorpb.MethodDescriptorProto: - descriptorType = "method" - default: - descriptorType = fmt.Sprintf("%T", d) - } - return fmt.Errorf("%s is unsupported filter type: %s", fullName, descriptorType) -} - -func (t *transitiveClosure) addEnclosing(descriptor namedDescriptor, enclosingFile string, imageIndex *imageIndex, opts *imageFilterOptions) error { - // loop through all enclosing parents since nesting level - // could be arbitrarily deep - for descriptor != nil { - _, isMsg := descriptor.(*descriptorpb.DescriptorProto) - _, isSvc := descriptor.(*descriptorpb.ServiceDescriptorProto) - if !isMsg && !isSvc { - break // not an enclosing type - } - if _, ok := t.elements[descriptor]; ok { - break // already in closure - } - t.elements[descriptor] = inclusionModeEnclosing - if err := t.exploreCustomOptions(descriptor, enclosingFile, imageIndex, opts); err != nil { - return err - } - // now move into this element's parent - descriptor = imageIndex.ByDescriptor[descriptor].parent - } - return nil -} - -func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProto, referrerFile string, imageIndex *imageIndex, opts *imageFilterOptions) error { - switch field.GetType() { - case descriptorpb.FieldDescriptorProto_TYPE_ENUM, - descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, - descriptorpb.FieldDescriptorProto_TYPE_GROUP: - typeName := strings.TrimPrefix(field.GetTypeName(), ".") - typeDescriptor, ok := imageIndex.NameToDescriptor[typeName] - if !ok { - return fmt.Errorf("missing %q", typeName) - } - err := t.addElement(typeDescriptor, referrerFile, false, imageIndex, opts) - if err != nil { - return err - } - case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, - descriptorpb.FieldDescriptorProto_TYPE_FLOAT, - descriptorpb.FieldDescriptorProto_TYPE_INT64, - descriptorpb.FieldDescriptorProto_TYPE_UINT64, - descriptorpb.FieldDescriptorProto_TYPE_INT32, - descriptorpb.FieldDescriptorProto_TYPE_FIXED64, - descriptorpb.FieldDescriptorProto_TYPE_FIXED32, - descriptorpb.FieldDescriptorProto_TYPE_BOOL, - descriptorpb.FieldDescriptorProto_TYPE_STRING, - descriptorpb.FieldDescriptorProto_TYPE_BYTES, - descriptorpb.FieldDescriptorProto_TYPE_UINT32, - descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, - descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, - descriptorpb.FieldDescriptorProto_TYPE_SINT32, - descriptorpb.FieldDescriptorProto_TYPE_SINT64: - // nothing to follow, custom options handled below. - default: - return fmt.Errorf("unknown field type %d", field.GetType()) - } - return nil -} - -func (t *transitiveClosure) addExtensions( - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - if !opts.includeKnownExtensions { - return nil // nothing to do - } - for e, mode := range t.elements { - if mode != inclusionModeExplicit { - // we only collect extensions for messages that are directly reachable/referenced. - continue - } - msgDescriptor, ok := e.(*descriptorpb.DescriptorProto) - if !ok { - // not a message, nothing to do - continue - } - descriptorInfo := imageIndex.ByDescriptor[msgDescriptor] - for _, extendsDescriptor := range imageIndex.NameToExtensions[descriptorInfo.fullName] { - if err := t.addElement(extendsDescriptor, "", false, imageIndex, opts); err != nil { - return err - } - } - } - return nil -} - -func (t *transitiveClosure) exploreCustomOptions( - descriptor proto.Message, - referrerFile string, - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - if !opts.includeCustomOptions { - return nil - } - - var options protoreflect.Message - switch descriptor := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.DescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.FieldDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.OneofDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.EnumDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.EnumValueDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.ServiceDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.MethodDescriptorProto: - options = descriptor.GetOptions().ProtoReflect() - case *descriptorpb.DescriptorProto_ExtensionRange: - options = descriptor.GetOptions().ProtoReflect() - default: - return fmt.Errorf("unexpected type for exploring options %T", descriptor) - } - - optionsName := string(options.Descriptor().FullName()) - var err error - options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - // If the value contains an Any message, we should add the message type - // therein to the closure. - if err = t.exploreOptionValueForAny(fd, val, referrerFile, imageIndex, opts); err != nil { - return false - } - - // Also include custom option definitions (e.g. extensions) - if !fd.IsExtension() { - return true - } - optionsByNumber := imageIndex.NameToOptions[optionsName] - field, ok := optionsByNumber[int32(fd.Number())] - if !ok { - err = fmt.Errorf("cannot find ext no %d on %s", fd.Number(), optionsName) - return false - } - err = t.addElement(field, referrerFile, true, imageIndex, opts) - return err == nil - }) - return err -} - -func isMessageKind(k protoreflect.Kind) bool { - return k == protoreflect.MessageKind || k == protoreflect.GroupKind -} - -func (t *transitiveClosure) exploreOptionValueForAny( - fd protoreflect.FieldDescriptor, - val protoreflect.Value, - referrerFile string, - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - switch { - case fd.IsMap(): - if isMessageKind(fd.MapValue().Kind()) { - var err error - val.Map().Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool { - if err = t.exploreOptionSingularValueForAny(v.Message(), referrerFile, imageIndex, opts); err != nil { - return false - } - return true - }) - return err - } - case isMessageKind(fd.Kind()): - if fd.IsList() { - listVal := val.List() - for i := 0; i < listVal.Len(); i++ { - if err := t.exploreOptionSingularValueForAny(listVal.Get(i).Message(), referrerFile, imageIndex, opts); err != nil { - return err - } - } - } else { - return t.exploreOptionSingularValueForAny(val.Message(), referrerFile, imageIndex, opts) - } - } - return nil -} - -func (t *transitiveClosure) exploreOptionSingularValueForAny( - msg protoreflect.Message, - referrerFile string, - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - md := msg.Descriptor() - if md.FullName() == anyFullName { - // Found one! - typeURLFd := md.Fields().ByNumber(1) - if typeURLFd.Kind() != protoreflect.StringKind || typeURLFd.IsList() { - // should not be possible... - return nil - } - typeURL := msg.Get(typeURLFd).String() - pos := strings.LastIndexByte(typeURL, '/') - msgType := typeURL[pos+1:] - d, _ := imageIndex.NameToDescriptor[msgType].(*descriptorpb.DescriptorProto) - if d != nil { - if err := t.addElement(d, referrerFile, false, imageIndex, opts); err != nil { - return err - } - } - // TODO: unmarshal the bytes to see if there are any nested Any messages - return nil - } - // keep digging - var err error - msg.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - err = t.exploreOptionValueForAny(fd, val, referrerFile, imageIndex, opts) - return err == nil - }) - return err -}*/ - type int32Range struct { start, end int32 // both inclusive } diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go similarity index 63% rename from private/bufpkg/bufimage/bufimageutil/exclude_options.go rename to private/bufpkg/bufimage/bufimageutil/image_filter.go index 2db0dc8003..a5c3895f04 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -15,7 +15,6 @@ package bufimageutil import ( - "encoding/json" "fmt" "slices" "sort" @@ -30,20 +29,11 @@ import ( // filterImage filters the Image for the given options. func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Image, error) { - imageTypeIndex, err := newImageTypeIndex(image) + imageIndex, err := newImageIndexForImage(image, options) if err != nil { return nil, err } - b, _ := json.MarshalIndent(imageTypeIndex.TypeSet, "", " ") - fmt.Println("index:", string(b)) - typeFilter, err := createTypeFilter(imageTypeIndex, options) - if err != nil { - return nil, err - } - fmt.Println("\ttypeFilter:", typeFilter) - fmt.Println("\t\tincludes:", typeFilter.include) - fmt.Println("\t\texcludes:", typeFilter.exclude) - optionsFilter, err := createOptionsFilter(imageTypeIndex, options) + filter, err := newFullNameFilter(imageIndex, options) if err != nil { return nil, err } @@ -65,9 +55,10 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im } newImageFile, err := filterImageFile( imageFile, - imageTypeIndex, - typeFilter, - optionsFilter, + imageIndex, + filter, + //typeFilter, + //optionsFilter, ) if err != nil { return nil, err @@ -91,18 +82,18 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im func filterImageFile( imageFile bufimage.ImageFile, - imageTypeIndex *imageTypeIndex, - typesFilter fullNameFilter, - optionsFilter fullNameFilter, + imageIndex *imageIndex, + filter *fullNameFilter, + //typesFilter fullNameFilter, + //optionsFilter fullNameFilter, ) (bufimage.ImageFile, error) { fileDescriptor := imageFile.FileDescriptorProto() var sourcePathsRemap sourcePathsRemapTrie isIncluded, err := addRemapsForFileDescriptor( &sourcePathsRemap, fileDescriptor, - imageTypeIndex, - typesFilter, - optionsFilter, + imageIndex, + filter, ) if err != nil { return nil, err @@ -130,26 +121,24 @@ func filterImageFile( } type sourcePathsBuilder struct { - filePath string - imageTypeIndex *imageTypeIndex - typesFilter fullNameFilter - optionsFilter fullNameFilter - fileImports map[string]struct{} + filePath string + imageIndex *imageIndex + filter *fullNameFilter + fileImports map[string]struct{} } func addRemapsForFileDescriptor( sourcePathsRemap *sourcePathsRemapTrie, fileDescriptor *descriptorpb.FileDescriptorProto, - imageTypeIndex *imageTypeIndex, - typesFilter fullNameFilter, - optionsFilter fullNameFilter, + imageIndex *imageIndex, + filter *fullNameFilter, + //typesFilter fullNameFilter, + //optionsFilter fullNameFilter, ) (bool, error) { - fmt.Println("---", fileDescriptor.GetName(), "---") - defer fmt.Println("------") packageName := protoreflect.FullName(fileDescriptor.GetPackage()) if packageName != "" { // Check if filtered by the package name. - isIncluded, isExplicit := typesFilter.filter(packageName) + isIncluded, isExplicit := filter.hasType(packageName) if !isIncluded && isExplicit { // The package is excluded. return false, nil @@ -158,11 +147,10 @@ func addRemapsForFileDescriptor( fileImports := make(map[string]struct{}) builder := &sourcePathsBuilder{ - filePath: fileDescriptor.GetName(), - imageTypeIndex: imageTypeIndex, - typesFilter: typesFilter, - optionsFilter: optionsFilter, - fileImports: fileImports, + filePath: fileDescriptor.GetName(), + imageIndex: imageIndex, + filter: filter, + fileImports: fileImports, } sourcePath := make(protoreflect.SourcePath, 0, 8) @@ -185,10 +173,9 @@ func addRemapsForFileDescriptor( // Fix the imports to remove any that are no longer used. // TODO: handle unused dependencies, and keep them? - fmt.Println("fileImports", builder.fileImports) if len(fileImports) != len(fileDescriptor.Dependency) { indexTo := int32(0) - dependencyPath := []int32{fileDependencyTag} + dependencyPath := append(sourcePath, fileDependencyTag) dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) for indexFrom, dependency := range fileDescriptor.Dependency { path := append(dependencyPath, int32(indexFrom)) @@ -203,7 +190,7 @@ func addRemapsForFileDescriptor( dependencyChanges[indexFrom] = -1 } } - publicDependencyPath := []int32{filePublicDependencyTag} + publicDependencyPath := append(sourcePath, filePublicDependencyTag) for indexFrom, publicDependency := range fileDescriptor.PublicDependency { path := append(publicDependencyPath, int32(indexFrom)) indexTo := dependencyChanges[publicDependency] @@ -213,7 +200,7 @@ func addRemapsForFileDescriptor( sourcePathsRemap.markMoved(path, indexTo) } } - weakDependencyPath := []int32{fileWeakDependencyTag} + weakDependencyPath := append(sourcePath, fileWeakDependencyTag) for indexFrom, weakDependency := range fileDescriptor.WeakDependency { path := append(weakDependencyPath, int32(indexFrom)) indexTo := dependencyChanges[weakDependency] @@ -234,38 +221,39 @@ func (b *sourcePathsBuilder) addRemapsForDescriptor( descriptor *descriptorpb.DescriptorProto, ) (bool, error) { fullName := getFullName(parentName, descriptor) - isIncluded, isExplicit := b.typesFilter.filter(fullName) + isIncluded, isExplicit := b.filter.hasType(fullName) if !isIncluded && isExplicit { // The type is excluded. return false, nil } - //// If the message is only enclosing an included message remove the fields. - //if isIncluded { - // if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { - // return false, err - // } - // if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { - // return false, err - // } - // for index, extensionRange := range descriptor.GetExtensionRange() { - // fmt.Println("\textensionRange", index, extensionRange) - // extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) - // if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { - // return false, err - // } - // } - // if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { - // return false, err - // } - //} else { - // sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) - // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionsTag)) - // for index := range descriptor.GetExtensionRange() { - // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag)) - // } - // sourcePathsRemap.markDeleted(append(sourcePath, messageOptionsTag)) - //} - + // + // If the message is only enclosin included message remove the fields. + if isIncluded { + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { + return false, err + } + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { + return false, err + } + for index, extensionRange := range descriptor.GetExtensionRange() { + extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) + if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { + return false, err + } + } + } else { + sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) + // TODO: check if extensions are removed??? + sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageReservedRangesTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageReservedNamesTag)) + //for index := range descriptor.GetExtensionRange() { + // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag)) + //} + //sourcePathsRemap.markDeleted(append(sourcePath, messageOptionsTag)) + } // Walk the nested types. hasNestedTypes, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.addRemapsForDescriptor) if err != nil { @@ -287,26 +275,9 @@ func (b *sourcePathsBuilder) addRemapsForDescriptor( } isIncluded = isIncluded || hasOneofs - // If the message is only enclosing an included message remove the fields. - if isIncluded { - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { - return false, err - } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { - return false, err - } - for index, extensionRange := range descriptor.GetExtensionRange() { - fmt.Println("\textensionRange", index, extensionRange) - extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) - if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { - return false, err - } - } - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { - return false, err - } - } else { - + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { + return false, err + } return isIncluded, nil } @@ -316,8 +287,9 @@ func (b *sourcePathsBuilder) addRemapsForEnum( sourcePath protoreflect.SourcePath, enum *descriptorpb.EnumDescriptorProto, ) (bool, error) { + //fullName := b.imageIndex.ByDescriptor[enum] fullName := getFullName(parentName, enum) - if isIncluded, _ := b.typesFilter.filter(fullName); !isIncluded { + if isIncluded, _ := b.filter.hasType(fullName); !isIncluded { // The type is excluded, enum values cannot be excluded individually. return false, nil } @@ -344,7 +316,7 @@ func (b *sourcePathsBuilder) addRemapsForOneof( oneof *descriptorpb.OneofDescriptorProto, ) (bool, error) { fullName := getFullName(parentName, oneof) - if isIncluded, _ := b.typesFilter.filter(fullName); !isIncluded { + if isIncluded, _ := b.filter.hasType(fullName); !isIncluded { // The type is excluded, enum values cannot be excluded individually. return false, nil } @@ -361,7 +333,7 @@ func (b *sourcePathsBuilder) addRemapsForService( service *descriptorpb.ServiceDescriptorProto, ) (bool, error) { fullName := getFullName(parentName, service) - isIncluded, isExplicit := b.typesFilter.filter(fullName) + isIncluded, isExplicit := b.filter.hasType(fullName) if !isIncluded && isExplicit { // The type is excluded. return false, nil @@ -386,18 +358,18 @@ func (b *sourcePathsBuilder) addRemapsForMethod( method *descriptorpb.MethodDescriptorProto, ) (bool, error) { fullName := getFullName(parentName, method) - if isIncluded, _ := b.typesFilter.filter(fullName); !isIncluded { + if isIncluded, _ := b.filter.hasType(fullName); !isIncluded { // The type is excluded. return false, nil } inputName := protoreflect.FullName(strings.TrimPrefix(method.GetInputType(), ".")) - if isIncluded, _ := b.typesFilter.filter(inputName); !isIncluded { + if isIncluded, _ := b.filter.hasType(inputName); !isIncluded { // The input type is excluded. return false, fmt.Errorf("input type %s of method %s is excluded", inputName, fullName) } b.addRequiredType(inputName) outputName := protoreflect.FullName(strings.TrimPrefix(method.GetOutputType(), ".")) - if isIncluded, _ := b.typesFilter.filter(outputName); !isIncluded { + if isIncluded, _ := b.filter.hasType(outputName); !isIncluded { // The output type is excluded. return false, fmt.Errorf("output type %s of method %s is excluded", outputName, fullName) } @@ -417,7 +389,7 @@ func (b *sourcePathsBuilder) addRemapsForField( if field.Extendee != nil { // This is an extension field. extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) - if isIncluded, _ := b.typesFilter.filter(extendeeName); !isIncluded { + if isIncluded, _ := b.filter.hasType(extendeeName); !isIncluded { return false, nil } b.addRequiredType(extendeeName) @@ -427,7 +399,7 @@ func (b *sourcePathsBuilder) addRemapsForField( descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, descriptorpb.FieldDescriptorProto_TYPE_GROUP: typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) - if isIncluded, _ := b.typesFilter.filter(typeName); !isIncluded { + if isIncluded, _ := b.filter.hasType(typeName); !isIncluded { return false, nil } b.addRequiredType(typeName) @@ -465,16 +437,15 @@ func (b *sourcePathsBuilder) addRemapsForOptions( } options := optionsMessage.ProtoReflect() numFieldsToKeep := 0 - options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - isIncluded, _ := b.optionsFilter.filter(fd.FullName()) + options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { + + isIncluded, _ := b.filter.hasOption(fd.FullName(), fd.IsExtension()) if !isIncluded { // Remove this option. - fmt.Println("\tremove option", fd.FullName()) optionPath := append(optionsPath, int32(fd.Number())) sourcePathsRemap.markDeleted(optionPath) return true } - fmt.Println("\tkeep option", fd.FullName()) numFieldsToKeep++ if fd.IsExtension() { // Add the extension type to the required types. @@ -490,14 +461,13 @@ func (b *sourcePathsBuilder) addRemapsForOptions( } func (b *sourcePathsBuilder) addRequiredType(fullName protoreflect.FullName) { - file, ok := b.imageTypeIndex.TypeToFile[fullName] + info, ok := b.imageIndex.ByName[fullName] if !ok { panic(fmt.Sprintf("could not find file for %s", fullName)) } - fmt.Println("\taddRequiredType", fullName, file.Path()) + file := info.imageFile if file.Path() != b.filePath { // This is an imported type. - file := b.imageTypeIndex.TypeToFile[fullName] b.fileImports[file.Path()] = struct{}{} } } @@ -533,213 +503,6 @@ func addRemapsForSlice[T any]( return toIndex > 0, nil } -/* -type fileDescriptorWalker struct { - filePath string - imageTypeIndex *imageTypeIndex - typesFilter fullNameFilter - optionsFilter fullNameFilter - - // On walking record whether the type is inlcuded. - includes []bool - - sourcePathsRemap sourcePathsRemapTrie - fileImports map[string]struct{} -} - -func (f *fileDescriptorWalker) walkFile( - fileDescriptor *descriptorpb.FileDescriptorProto, - sourcePathsRemap sourcePathsRemapTrie, -) error { - prefix := fileDescriptor.GetPackage() - if prefix != "" { - prefix += "." - } - isIncluded, isExplicit := f.typesFilter.filter(protoreflect.FullName(prefix)) - if !isIncluded && isExpli - - sourcePath := make(protoreflect.SourcePath, 0, 16) - toIndex := 0 - for fromIndex := range fileDescriptor.MessageType { - sourcePath := append(sourcePath, fileMessagesTag, int32(fromIndex)) - isIncluded, err := f.walkMessage(prefix, sourcePath, fileDescriptor.MessageType[fromIndex]) - if err != nil { - return err - } - if isIncluded && fromIndex != toIndex { - f.sourcePathsRemap.markMoved(sourcePath, int32(toIndex)) - } else { - f.sourcePathsRemap.markDeleted(sourcePath) - } - } -} - -func (f *fileDescriptorWalker) walkMessage(prefix string, sourcePath protoreflect.SourcePath, descriptor *descriptorpb.DescriptorProto) (bool, error) { - -} - -func (f *fileDescriptorWalker) enter(fullName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor proto.Message) error { - var isIncluded bool - switch descriptor := descriptor.(type) { - case *descriptorpb.EnumValueDescriptorProto, *descriptorpb.OneofDescriptorProto: - // Added by their enclosing types. - isIncluded = f.includes[len(f.includes)-1] - case *descriptorpb.FieldDescriptorProto: - isIncluded = f.isFieldIncluded(descriptor) - default: - isIncluded = f.typesFilter.filter(fullName) - } - fmt.Println("ENTER", fullName, "included?", isIncluded) - if isIncluded { - // If a child is included, the parent must be included. - for index := range f.includes { - f.includes[index] = true - } - } - f.includes = append(f.includes, isIncluded) - return nil -} - -func (f *fileDescriptorWalker) exit(fullName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor proto.Message) error { - fmt.Println("EXIT", f.includes) - isIncluded := f.includes[len(f.includes)-1] - f.includes = f.includes[:len(f.includes)-1] - if !isIncluded { - // Mark the source path for deletion. - f.sourcePathsRemap.markDeleted(sourcePath) - fmt.Println("\tDELETE", fullName) - return nil - } - // If the type is included, walk the options. - switch descriptor := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - // File options are handled at the top level, before walking the file. - // The FileDescriptorProto is not walked here. - return nil - case *descriptorpb.DescriptorProto: - optionsPath := append(sourcePath, messageOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.FieldDescriptorProto: - // Add the field type to the required types. - if err := f.addFieldType(descriptor); err != nil { - return err - } - optionsPath := append(sourcePath, fieldOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.OneofDescriptorProto: - optionsPath := append(sourcePath, oneofOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.EnumDescriptorProto: - optionsPath := append(sourcePath, enumOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.EnumValueDescriptorProto: - optionsPath := append(sourcePath, enumValueOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.ServiceDescriptorProto: - optionsPath := append(sourcePath, serviceOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.MethodDescriptorProto: - optionsPath := append(sourcePath, methodOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - case *descriptorpb.DescriptorProto_ExtensionRange: - optionsPath := append(sourcePath, extensionRangeOptionsTag) - return f.options(descriptor.GetOptions(), optionsPath) - default: - return fmt.Errorf("unexpected message type %T", descriptor) - } -} - -func (f *fileDescriptorWalker) options( - optionsMessage proto.Message, - optionsPath protoreflect.SourcePath, -) error { - if optionsMessage == nil { - return nil - } - options := optionsMessage.ProtoReflect() - if !options.IsValid() { - return nil // No options to strip. - } - numFieldsToKeep := 0 - options.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - if !f.optionsFilter.filter(fd.FullName()) { - // Remove this option. - fmt.Println("\tremove option", fd.FullName()) - optionPath := append(optionsPath, int32(fd.Number())) - f.sourcePathsRemap.markDeleted(optionPath) - return true - } - fmt.Println("\tkeep option", fd.FullName()) - numFieldsToKeep++ - if fd.IsExtension() { - // Add the extension type to the required types. - f.addRequiredType(fd.FullName()) - } - return true - }) - if numFieldsToKeep == 0 { - f.sourcePathsRemap.markDeleted(optionsPath) // No options to keep. - } - return nil -} - -func (f *fileDescriptorWalker) isFieldIncluded(field *descriptorpb.FieldDescriptorProto) bool { - isIncluded := f.includes[len(f.includes)-1] - if field.Extendee != nil { - // This is an extension field. - extendee := strings.TrimPrefix(field.GetExtendee(), ".") - isIncluded = isIncluded && f.typesFilter.filter(protoreflect.FullName(extendee)) - } - typeName := strings.TrimPrefix(field.GetTypeName(), ".") - isIncluded = isIncluded && f.typesFilter.filter(protoreflect.FullName(typeName)) - fmt.Println("\tisFieldIncluded", field.GetName(), typeName, isIncluded) - return isIncluded -} - -func (f *fileDescriptorWalker) addFieldType(field *descriptorpb.FieldDescriptorProto) error { - if field.Extendee != nil { - // This is an extension field. - extendee := strings.TrimPrefix(field.GetExtendee(), ".") - f.addRequiredType(protoreflect.FullName(extendee)) - } - switch field.GetType() { - case descriptorpb.FieldDescriptorProto_TYPE_ENUM, - descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, - descriptorpb.FieldDescriptorProto_TYPE_GROUP: - typeName := strings.TrimPrefix(field.GetTypeName(), ".") - f.addRequiredType(protoreflect.FullName(typeName)) - case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, - descriptorpb.FieldDescriptorProto_TYPE_FLOAT, - descriptorpb.FieldDescriptorProto_TYPE_INT64, - descriptorpb.FieldDescriptorProto_TYPE_UINT64, - descriptorpb.FieldDescriptorProto_TYPE_INT32, - descriptorpb.FieldDescriptorProto_TYPE_FIXED64, - descriptorpb.FieldDescriptorProto_TYPE_FIXED32, - descriptorpb.FieldDescriptorProto_TYPE_BOOL, - descriptorpb.FieldDescriptorProto_TYPE_STRING, - descriptorpb.FieldDescriptorProto_TYPE_BYTES, - descriptorpb.FieldDescriptorProto_TYPE_UINT32, - descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, - descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, - descriptorpb.FieldDescriptorProto_TYPE_SINT32, - descriptorpb.FieldDescriptorProto_TYPE_SINT64: - // nothing to follow, custom options handled above. - default: - return fmt.Errorf("unknown field type %d", field.GetType()) - } - return nil -} - -func (f *fileDescriptorWalker) addRequiredType(fullName protoreflect.FullName) { - file := f.imageTypeIndex.TypeToFile[fullName] - fmt.Println("\taddRequiredType", fullName, file.Path()) - if file.Path() != f.filePath { - // This is an imported type. - file := f.imageTypeIndex.TypeToFile[fullName] - f.fileImports[file.Path()] = struct{}{} - } -}*/ - func remapFileDescriptor( fileDescriptor *descriptorpb.FileDescriptorProto, sourcePathRemaps sourcePathsRemapTrie, @@ -929,11 +692,3 @@ func shallowCloneReflect(src protoreflect.Message) protoreflect.Message { }) return dst } - -func getFullName(parentName protoreflect.FullName, message interface{ GetName() string }) protoreflect.FullName { - fullName := protoreflect.FullName(message.GetName()) - if parentName == "" { - return fullName - } - return parentName + "." + fullName -} diff --git a/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go b/private/bufpkg/bufimage/bufimageutil/image_filter_test.go similarity index 87% rename from private/bufpkg/bufimage/bufimageutil/exclude_options_test.go rename to private/bufpkg/bufimage/bufimageutil/image_filter_test.go index 45b64c5e35..2d0c67c7f6 100644 --- a/private/bufpkg/bufimage/bufimageutil/exclude_options_test.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter_test.go @@ -40,13 +40,13 @@ func TestExcludeOptions(t *testing.T) { t.Run("NoneExcluded", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", ) }) t.Run("ExcludeMessage", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", WithExcludeOptions( "message_foo", "message_bar", "message_baz", ), @@ -54,7 +54,7 @@ func TestExcludeOptions(t *testing.T) { }) t.Run("ExcludeFoo", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", WithExcludeOptions( "message_foo", "field_foo", @@ -69,7 +69,7 @@ func TestExcludeOptions(t *testing.T) { }) t.Run("OnlyFile", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", WithExcludeOptions( "message_foo", "message_bar", "message_baz", "field_foo", "field_bar", "field_baz", @@ -83,7 +83,7 @@ func TestExcludeOptions(t *testing.T) { }) t.Run("OnlyOneOf", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", WithExcludeOptions( "message_foo", "message_bar", "message_baz", "field_foo", "field_bar", "field_baz", @@ -97,7 +97,7 @@ func TestExcludeOptions(t *testing.T) { }) t.Run("OnlyEnumValue", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", WithExcludeOptions( "message_foo", "message_bar", "message_baz", "field_foo", "field_bar", "field_baz", @@ -111,7 +111,7 @@ func TestExcludeOptions(t *testing.T) { }) t.Run("ExcludeAll", func(t *testing.T) { t.Parallel() - testExcludeOptions( + testFilterOptions( t, "testdata/excludeoptions", WithExcludeOptions( "message_foo", "message_bar", "message_baz", "field_foo", "field_bar", "field_baz", @@ -157,19 +157,19 @@ func TestExcludeOptionImports(t *testing.T) { t.Run("NoneExcluded", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image) + testFilterOptionsForImage(t, bucket, image) }) t.Run("ExcludeFoo", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo")) + testFilterOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo")) }) t.Run("ExcludeFooBar", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo", "message_bar")) + testFilterOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo", "message_bar")) }) t.Run("ExcludeBar", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, WithExcludeOptions("message_bar")) + testFilterOptionsForImage(t, bucket, image, WithExcludeOptions("message_bar")) }) } @@ -202,22 +202,22 @@ func TestFilterTypes(t *testing.T) { t.Run("ExcludeBar", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, WithExcludeTypes("pkg.Bar")) + testFilterOptionsForImage(t, bucket, image, WithExcludeTypes("pkg.Bar")) }) t.Run("IncludeBar", func(t *testing.T) { t.Parallel() - testExcludeOptionsForImage(t, bucket, image, WithIncludeTypes("pkg.Bar")) + testFilterOptionsForImage(t, bucket, image, WithIncludeTypes("pkg.Bar")) }) } -func testExcludeOptions(t *testing.T, testdataDir string, options ...ImageFilterOption) { +func testFilterOptions(t *testing.T, testdataDir string, options ...ImageFilterOption) { bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) require.NoError(t, err) - testExcludeOptionsForModuleData(t, bucket, nil, options...) + testFilterOptionsForModuleData(t, bucket, nil, options...) } -func testExcludeOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucket, moduleData []bufmoduletesting.ModuleData, options ...ImageFilterOption) { +func testFilterOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucket, moduleData []bufmoduletesting.ModuleData, options ...ImageFilterOption) { ctx := context.Background() if len(moduleData) == 0 { moduleData = append(moduleData, bufmoduletesting.ModuleData{ @@ -235,10 +235,10 @@ func testExcludeOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucke ) require.NoError(t, err) - testExcludeOptionsForImage(t, bucket, image, options...) + testFilterOptionsForImage(t, bucket, image, options...) } -func testExcludeOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, image bufimage.Image, options ...ImageFilterOption) { +func testFilterOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, image bufimage.Image, options ...ImageFilterOption) { ctx := context.Background() filteredImage, err := FilterImage(image, options...) require.NoError(t, err) diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 0eb7c33a30..575f8b991f 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -25,8 +25,6 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -// file -> DAG types. - // imageIndex holds an index that allows for easily navigating a descriptor // hierarchy and its relationships. type imageIndex struct { @@ -36,9 +34,8 @@ type imageIndex struct { ByDescriptor map[namedDescriptor]elementInfo // ByName maps fully qualified type names to information about the named // element. - ByName map[string]namedDescriptor - // Files maps fully qualified type names to the path of the file that - // declares the type. + ByName map[protoreflect.FullName]elementInfo + // Files maps file names to the file descriptor protos. Files map[string]*descriptorpb.FileDescriptorProto // NameToExtensions maps fully qualified type names to all known @@ -50,7 +47,7 @@ type imageIndex struct { NameToOptions map[string]map[int32]*descriptorpb.FieldDescriptorProto // Packages maps package names to package contents. - Packages map[string]*protoPackage + Packages map[string]*packageInfo } type namedDescriptor interface { @@ -68,28 +65,31 @@ var _ namedDescriptor = (*descriptorpb.ServiceDescriptorProto)(nil) var _ namedDescriptor = (*descriptorpb.MethodDescriptorProto)(nil) type elementInfo struct { - fullName, file string - parent namedDescriptor + fullName protoreflect.FullName + imageFile bufimage.ImageFile //string // TODO: maybe bufimage.ImageFile? + parentName protoreflect.FullName //namedDescriptor + element namedDescriptor } -type protoPackage struct { +type packageInfo struct { + fullName protoreflect.FullName files []bufimage.ImageFile - elements []namedDescriptor - subPackages []*protoPackage + types []protoreflect.FullName + subPackages []*packageInfo } // newImageIndexForImage builds an imageIndex for a given image. -func newImageIndexForImage(image bufimage.Image, opts *imageFilterOptions) (*imageIndex, error) { +func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (*imageIndex, error) { index := &imageIndex{ - ByName: make(map[string]namedDescriptor), + ByName: make(map[protoreflect.FullName]elementInfo), ByDescriptor: make(map[namedDescriptor]elementInfo), Files: make(map[string]*descriptorpb.FileDescriptorProto), - Packages: make(map[string]*protoPackage), + Packages: make(map[string]*packageInfo), } - if opts.includeCustomOptions { + if options.includeCustomOptions { index.NameToOptions = make(map[string]map[int32]*descriptorpb.FieldDescriptorProto) } - if opts.includeKnownExtensions { + if options.includeKnownExtensions { index.NameToExtensions = make(map[string][]*descriptorpb.FieldDescriptorProto) } @@ -100,20 +100,16 @@ func newImageIndexForImage(image bufimage.Image, opts *imageFilterOptions) (*ima fileDescriptorProto := imageFile.FileDescriptorProto() index.Files[fileName] = fileDescriptorProto err := walk.DescriptorProtos(fileDescriptorProto, func(name protoreflect.FullName, msg proto.Message) error { - if existing := index.ByName[string(name)]; existing != nil { + if _, existing := index.ByName[name]; existing { return fmt.Errorf("duplicate for %q", name) } descriptor, ok := msg.(namedDescriptor) if !ok { return fmt.Errorf("unexpected descriptor type %T", msg) } - var parent namedDescriptor + var parentName protoreflect.FullName if pos := strings.LastIndexByte(string(name), '.'); pos != -1 { - parent = index.ByName[string(name[:pos])] - if parent == nil { - // parent name was a package name, not an element name - parent = fileDescriptorProto - } + parentName = name[:pos] } // certain descriptor types don't need to be indexed: @@ -130,13 +126,15 @@ func newImageIndexForImage(image bufimage.Image, opts *imageFilterOptions) (*ima } if includeInIndex { - index.ByName[string(name)] = descriptor - index.ByDescriptor[descriptor] = elementInfo{ - fullName: string(name), - parent: parent, - file: fileName, + info := elementInfo{ + fullName: name, + imageFile: imageFile, + parentName: parentName, + element: descriptor, } - pkg.elements = append(pkg.elements, descriptor) + index.ByName[name] = info + index.ByDescriptor[descriptor] = info + pkg.types = append(pkg.types, name) } ext, ok := descriptor.(*descriptorpb.FieldDescriptorProto) @@ -146,16 +144,15 @@ func newImageIndexForImage(image bufimage.Image, opts *imageFilterOptions) (*ima } extendeeName := strings.TrimPrefix(ext.GetExtendee(), ".") - if opts.includeCustomOptions && isOptionsTypeName(extendeeName) { + if options.includeCustomOptions && isOptionsTypeName(extendeeName) { if _, ok := index.NameToOptions[extendeeName]; !ok { index.NameToOptions[extendeeName] = make(map[int32]*descriptorpb.FieldDescriptorProto) } index.NameToOptions[extendeeName][ext.GetNumber()] = ext } - if opts.includeKnownExtensions { + if options.includeKnownExtensions { index.NameToExtensions[extendeeName] = append(index.NameToExtensions[extendeeName], ext) } - return nil }) if err != nil { @@ -165,12 +162,14 @@ func newImageIndexForImage(image bufimage.Image, opts *imageFilterOptions) (*ima return index, nil } -func addPackageToIndex(pkgName string, index *imageIndex) *protoPackage { +func addPackageToIndex(pkgName string, index *imageIndex) *packageInfo { pkg := index.Packages[pkgName] if pkg != nil { return pkg } - pkg = &protoPackage{} + pkg = &packageInfo{ + fullName: protoreflect.FullName(pkgName), + } index.Packages[pkgName] = pkg if pkgName == "" { return pkg @@ -200,3 +199,11 @@ func isOptionsTypeName(typeName string) bool { return false } } + +func getFullName(parentName protoreflect.FullName, descriptor namedDescriptor) protoreflect.FullName { + fullName := protoreflect.FullName(descriptor.GetName()) + if parentName == "" { + return fullName + } + return parentName + "." + fullName +} diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go index 4d2b1f81fa..80296ccd79 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_types.go +++ b/private/bufpkg/bufimage/bufimageutil/image_types.go @@ -18,288 +18,483 @@ import ( "fmt" "strings" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/protocompile/walk" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) -type typeLink int - -const ( - typeLinkNone typeLink = iota // 0 - typeLinkChild // 1 - typeLinkDepends // 2 - typeLinkOption // 3 -) - -type imageTypeIndex struct { - // TypeSet maps fully qualified type names to their children. - TypeSet map[protoreflect.FullName]map[protoreflect.FullName]typeLink - // TypeToFile maps fully qualified type names to their image file. - TypeToFile map[protoreflect.FullName]bufimage.ImageFile +type fullNameFilter struct { + options *imageFilterOptions + index *imageIndex + includes map[protoreflect.FullName]struct{} + excludes map[protoreflect.FullName]struct{} } -func newImageTypeIndex(image bufimage.Image) (*imageTypeIndex, error) { - index := &imageTypeIndex{ - TypeSet: make(map[protoreflect.FullName]map[protoreflect.FullName]typeLink), - TypeToFile: make(map[protoreflect.FullName]bufimage.ImageFile), +func newFullNameFilter( + imageIndex *imageIndex, + options *imageFilterOptions, +) (*fullNameFilter, error) { + filter := &fullNameFilter{ + options: options, + index: imageIndex, + } + if !options.includeCustomOptions && len(options.includeOptions) > 0 { + return nil, fmt.Errorf("cannot include options without including custom options") } - for _, file := range image.Files() { - if err := index.addFile(file); err != nil { + for excludeType := range options.excludeTypes { + excludeType := protoreflect.FullName(excludeType) + if err := filter.exclude(excludeType); err != nil { return nil, err } } - return index, nil -} - -func (i *imageTypeIndex) addFile(file bufimage.ImageFile) error { - fileDescriptor := file.FileDescriptorProto() - packageName := protoreflect.FullName(fileDescriptor.GetPackage()) - // Add all parent packages to point to this package. - for packageName := packageName; packageName != ""; { - sep := strings.LastIndex(string(packageName), ".") - if sep == -1 { - break - } - parentPackageName := protoreflect.FullName(packageName[:sep]) - i.linkType(parentPackageName, packageName, typeLinkChild) - packageName = parentPackageName + for includeType := range options.includeTypes { + includeType := protoreflect.FullName(includeType) + if err := filter.include(includeType); err != nil { + return nil, err + } } + return filter, nil +} - stack := make([]protoreflect.FullName, 0, 10) - stack = append(stack, packageName) - i.addType(stack[0]) - i.addOptionTypes(stack[0], fileDescriptor.GetOptions()) - enter := func(fullName protoreflect.FullName, descriptor proto.Message) error { - fmt.Println("stack", stack, "->", fullName) - parentFullName := stack[len(stack)-1] - switch descriptor := descriptor.(type) { - case *descriptorpb.DescriptorProto: - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.FieldDescriptorProto: - i.addFieldType(parentFullName, descriptor) - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.OneofDescriptorProto: - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.EnumDescriptorProto: - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.EnumValueDescriptorProto: - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.ServiceDescriptorProto: - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.MethodDescriptorProto: - inputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetInputType(), ".")) - outputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetOutputType(), ".")) - i.linkType(parentFullName, inputName, typeLinkDepends) - i.linkType(parentFullName, outputName, typeLinkDepends) - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - case *descriptorpb.DescriptorProto_ExtensionRange: - i.addOptionTypes(parentFullName, descriptor.GetOptions()) - default: - return fmt.Errorf("unexpected message type %T", descriptor) - } - i.TypeToFile[fullName] = file - if isDescriptorType(descriptor) { - i.linkType(parentFullName, fullName, typeLinkChild) - i.addType(fullName) - stack = append(stack, fullName) +func (f *fullNameFilter) hasType(fullName protoreflect.FullName) (isIncluded bool, isExplicit bool) { + if len(f.options.excludeTypes) > 0 || f.excludes != nil { + if _, ok := f.excludes[fullName]; ok { + return false, true } - return nil } - exit := func(fullName protoreflect.FullName, descriptor proto.Message) error { - if isDescriptorType(descriptor) { - stack = stack[:len(stack)-1] + if len(f.options.includeTypes) > 0 || f.includes != nil { + if _, ok := f.includes[fullName]; ok { + return true, true } - fmt.Println("exit ", stack, "->", fullName) - return nil - } - if err := walk.DescriptorProtosEnterAndExit(fileDescriptor, enter, exit); err != nil { - return err + return false, false } - return nil + return true, false } -func (i *imageTypeIndex) addType(fullName protoreflect.FullName) { - if _, ok := i.TypeSet[fullName]; !ok { - i.TypeSet[fullName] = nil +func (f *fullNameFilter) hasOption(fullName protoreflect.FullName, isExtension bool) (isIncluded bool, isExplicit bool) { + if f.options.excludeOptions != nil { + if _, ok := f.options.excludeOptions[string(fullName)]; ok { + return false, true + } } -} - -func (i *imageTypeIndex) linkType(parentFullName protoreflect.FullName, fullName protoreflect.FullName, link typeLink) { - if typeSet := i.TypeSet[parentFullName]; typeSet != nil { - typeSet[fullName] = link - } else { - i.TypeSet[parentFullName] = map[protoreflect.FullName]typeLink{fullName: link} + if !f.options.includeCustomOptions { + return !isExtension, true + } + if f.options.includeOptions != nil { + _, ok := f.options.includeOptions[string(fullName)] + return ok, true } + return true, false } -func (i *imageTypeIndex) addFieldType(parentFullName protoreflect.FullName, fieldDescriptor *descriptorpb.FieldDescriptorProto) { - if extendee := fieldDescriptor.GetExtendee(); extendee != "" { - // This is an extension field. - extendeeFullName := protoreflect.FullName(strings.TrimPrefix(extendee, ".")) - i.linkType(parentFullName, extendeeFullName, typeLinkDepends) - } - // Add the field type. - switch fieldDescriptor.GetType() { - case descriptorpb.FieldDescriptorProto_TYPE_ENUM, - descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, - descriptorpb.FieldDescriptorProto_TYPE_GROUP: - // Add links to the type of the field. - typeFullName := protoreflect.FullName( - strings.TrimPrefix(fieldDescriptor.GetTypeName(), "."), - ) - i.linkType(parentFullName, typeFullName, typeLinkDepends) +func (f *fullNameFilter) isExplicitExclude(fullName protoreflect.FullName) bool { + if f.excludes == nil { + return false } + _, ok := f.excludes[fullName] + return ok } -func (i *imageTypeIndex) addOptionTypes(parentFullName protoreflect.FullName, optionsMessage proto.Message) { - if optionsMessage == nil { - return +func (f *fullNameFilter) exclude(fullName protoreflect.FullName) error { + if _, excluded := f.excludes[fullName]; excluded { + return nil } - options := optionsMessage.ProtoReflect() - if !options.IsValid() { - return + if descriptorInfo, ok := f.index.ByName[fullName]; ok { + return f.excludeElement(fullName, descriptorInfo.element) } - options.Range(func(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) bool { - if fieldDescriptor.IsExtension() { - i.linkType(parentFullName, fieldDescriptor.FullName(), typeLinkOption) + packageInfo, ok := f.index.Packages[string(fullName)] + if !ok { + return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) + } + for _, file := range packageInfo.files { + // Remove the package name from the excludes since it is not unique per file. + delete(f.excludes, fullName) + if err := f.excludeElement(fullName, file.FileDescriptorProto()); err != nil { + return err } - return true - }) + } + for _, subPackage := range packageInfo.subPackages { + if err := f.exclude(subPackage.fullName); err != nil { + return err + } + } + return nil } -type fullNameFilter struct { - include map[protoreflect.FullName]struct{} - exclude map[protoreflect.FullName]struct{} +func (f *fullNameFilter) excludeElement(fullName protoreflect.FullName, descriptor namedDescriptor) error { + if _, excluded := f.excludes[fullName]; excluded { + return nil + } + if f.excludes == nil { + f.excludes = make(map[protoreflect.FullName]struct{}) + } + f.excludes[fullName] = struct{}{} + switch descriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + if err := forEachDescriptor(fullName, descriptor.GetMessageType(), f.excludeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.excludeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetService(), f.excludeElement); err != nil { + return err + } + return nil + case *descriptorpb.DescriptorProto: + f.excludes[fullName] = struct{}{} + // Exclude all sub-elements + if err := forEachDescriptor(fullName, descriptor.GetNestedType(), f.excludeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.excludeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetOneofDecl(), f.excludeElement); err != nil { + return err + } + return nil + case *descriptorpb.EnumDescriptorProto: + // Value is excluded by parent. + return nil + case *descriptorpb.OneofDescriptorProto: + return nil + case *descriptorpb.ServiceDescriptorProto: + if err := forEachDescriptor(fullName, descriptor.GetMethod(), f.excludeElement); err != nil { + return err + } + return nil + case *descriptorpb.MethodDescriptorProto: + return nil + default: + return errorUnsupportedFilterType(descriptor, fullName) + } } -func (f *fullNameFilter) filter(fullName protoreflect.FullName) (isIncluded bool, isExplicit bool) { - if f.exclude != nil { - if _, ok := f.exclude[fullName]; ok { - return false, true +func (f *fullNameFilter) include(fullName protoreflect.FullName) error { + if _, included := f.includes[fullName]; included { + return nil + } + if descriptorInfo, ok := f.index.ByName[fullName]; ok { + // Include the enclosing parent options. + if err := f.includeEnclosingOptions(descriptorInfo.parentName); err != nil { + return err } + return f.includeElement(fullName, descriptorInfo.element) } - if f.include != nil { - if _, ok := f.include[fullName]; ok { - return true, true + packageInfo, ok := f.index.Packages[string(fullName)] + if !ok { + return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) + } + for _, file := range packageInfo.files { + // Remove the package name from the includes since it is not unique per file. + delete(f.includes, fullName) + if err := f.includeElement(fullName, file.FileDescriptorProto()); err != nil { + return err } - return false, false } - return true, false + f.includes[fullName] = struct{}{} + for _, subPackage := range packageInfo.subPackages { + if err := f.include(subPackage.fullName); err != nil { + return err + } + } + return nil } -func createTypeFilter(index *imageTypeIndex, options *imageFilterOptions) (fullNameFilter, error) { - var filter fullNameFilter - var excludeList []protoreflect.FullName - for excludeType := range options.excludeTypes { - excludeType := protoreflect.FullName(excludeType) - if _, ok := index.TypeSet[excludeType]; !ok { - return filter, fmt.Errorf("filtering by excluded type %q: %w", excludeType, ErrImageFilterTypeNotFound) - } - file := index.TypeToFile[excludeType] - if file.IsImport() && !options.allowImportedTypes { - return filter, fmt.Errorf("filtering by excluded type %q: %w", excludeType, ErrImageFilterTypeIsImport) - } - excludeList = append(excludeList, excludeType) +func (f *fullNameFilter) includeElement(fullName protoreflect.FullName, descriptor namedDescriptor) error { + if _, included := f.includes[fullName]; included { + return nil + } + if f.isExplicitExclude(fullName) { + return nil // already excluded } - if len(excludeList) > 0 { - filter.exclude = make(map[protoreflect.FullName]struct{}) + if f.includes == nil { + f.includes = make(map[protoreflect.FullName]struct{}) + } + f.includes[fullName] = struct{}{} + + if err := f.includeOptions(descriptor); err != nil { + return err } - for len(excludeList) > 0 { - excludeType := excludeList[len(excludeList)-1] - excludeList = excludeList[:len(excludeList)-1] - if _, ok := filter.exclude[excludeType]; ok { - continue + + switch descriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + if err := forEachDescriptor(fullName, descriptor.GetMessageType(), f.includeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.includeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetService(), f.includeElement); err != nil { + return err + } + return nil + case *descriptorpb.DescriptorProto: + if err := forEachDescriptor(fullName, descriptor.GetNestedType(), f.includeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.includeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetOneofDecl(), f.includeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetField(), f.includeElement); err != nil { + return err + } + if err := forEachDescriptor(fullName, descriptor.GetExtension(), f.includeElement); err != nil { + return err + } + + // TODO: Include known extensions. + //if f.options.includeKnownExtensions { + // for _, extensionDescriptor := range f.index.NameToExtensions[string(fullName)] { + // if err := f.includeElement(fullName, extensionDescriptor); err != nil { + // return err + // } + // } + //} + return nil + case *descriptorpb.FieldDescriptorProto: + if descriptor.Extendee != nil { + // This is an extension field. + extendeeName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetExtendee(), ".")) + if err := f.include(extendeeName); err != nil { + return err + } } - for childType, childLink := range index.TypeSet[excludeType] { - if childLink == typeLinkChild { - excludeList = append(excludeList, childType) + switch descriptor.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_ENUM, + descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, + descriptorpb.FieldDescriptorProto_TYPE_GROUP: + typeName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetTypeName(), ".")) + if err := f.include(typeName); err != nil { + return err } + case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, + descriptorpb.FieldDescriptorProto_TYPE_FLOAT, + descriptorpb.FieldDescriptorProto_TYPE_INT64, + descriptorpb.FieldDescriptorProto_TYPE_UINT64, + descriptorpb.FieldDescriptorProto_TYPE_INT32, + descriptorpb.FieldDescriptorProto_TYPE_FIXED64, + descriptorpb.FieldDescriptorProto_TYPE_FIXED32, + descriptorpb.FieldDescriptorProto_TYPE_BOOL, + descriptorpb.FieldDescriptorProto_TYPE_STRING, + descriptorpb.FieldDescriptorProto_TYPE_BYTES, + descriptorpb.FieldDescriptorProto_TYPE_UINT32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, + descriptorpb.FieldDescriptorProto_TYPE_SINT32, + descriptorpb.FieldDescriptorProto_TYPE_SINT64: + default: + return fmt.Errorf("unknown field type %d", descriptor.GetType()) + } + return nil + case *descriptorpb.EnumDescriptorProto: + for _, enumValue := range descriptor.GetValue() { + if err := f.includeOptions(enumValue); err != nil { + return err + } + } + return nil + case *descriptorpb.OneofDescriptorProto: + return nil + case *descriptorpb.ServiceDescriptorProto: + if err := forEachDescriptor(fullName, descriptor.GetMethod(), f.includeElement); err != nil { + return err + } + return nil + case *descriptorpb.MethodDescriptorProto: + inputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetInputType(), ".")) + inputInfo, ok := f.index.ByName[inputName] + if !ok { + return fmt.Errorf("missing %q", inputName) + } + if err := f.includeElement(inputName, inputInfo.element); err != nil { + return err } - filter.exclude[excludeType] = struct{}{} - } - var includeList []protoreflect.FullName - for includeType := range options.includeTypes { - includeType := protoreflect.FullName(includeType) - if _, ok := index.TypeSet[includeType]; !ok { - return filter, fmt.Errorf("filtering by included type %q: %w", includeType, ErrImageFilterTypeNotFound) + outputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetOutputType(), ".")) + outputInfo, ok := f.index.ByName[outputName] + if !ok { + return fmt.Errorf("missing %q", outputName) } - file := index.TypeToFile[includeType] - if file.IsImport() && !options.allowImportedTypes { - return filter, fmt.Errorf("filtering by included type %q: %w", includeType, ErrImageFilterTypeIsImport) + if err := f.includeElement(outputName, outputInfo.element); err != nil { + return err } - if _, ok := filter.exclude[includeType]; ok { - continue // Skip already excluded. + return nil + default: + return errorUnsupportedFilterType(descriptor, fullName) + } +} + +func (f *fullNameFilter) includeEnclosingOptions(fullName protoreflect.FullName) error { + // loop through all enclosing parents since nesting level + // could be arbitrarily deep + for info, ok := f.index.ByName[fullName]; ok; { + if err := f.includeOptions(info.element); err != nil { + return err } - includeList = append(includeList, includeType) + info, ok = f.index.ByName[info.parentName] + } + return nil +} + +func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { + var optionsMessage proto.Message + switch descriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.DescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.FieldDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.OneofDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.EnumDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.EnumValueDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.ServiceDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.MethodDescriptorProto: + optionsMessage = descriptor.GetOptions() + case *descriptorpb.DescriptorProto_ExtensionRange: + optionsMessage = descriptor.GetOptions() + default: + return fmt.Errorf("unexpected type for exploring options %T", descriptor) } - if len(includeList) > 0 { - filter.include = make(map[protoreflect.FullName]struct{}) + if optionsMessage == nil { + return nil } - for len(includeList) > 0 { - includeType := includeList[len(includeList)-1] - includeList = includeList[:len(includeList)-1] - if _, ok := filter.include[includeType]; ok { - continue + options := optionsMessage.ProtoReflect() + optionsName := options.Descriptor().FullName() + optionsByNumber := f.index.NameToOptions[string(optionsName)] + options.Range(func(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) bool { + if isIncluded, _ := f.hasOption(fieldDescriptor.FullName(), fieldDescriptor.IsExtension()); !isIncluded { + return true } - for childType, childLink := range index.TypeSet[includeType] { - if _, ok := filter.exclude[includeType]; ok { - continue // Skip already excluded. - } - switch childLink { - case typeLinkChild: - includeList = append(includeList, childType) - case typeLinkDepends: - includeList = append(includeList, childType) - case typeLinkOption: - if options.includeKnownExtensions || (options.includeCustomOptions && isOptionsTypeName(string(childType))) { - includeList = append(includeList, childType) + if err = f.includeOptionValue(fieldDescriptor, value); err != nil { + return false + } + if !fieldDescriptor.IsExtension() { + return true + } + extensionField, ok := optionsByNumber[int32(fieldDescriptor.Number())] + if !ok { + err = fmt.Errorf("cannot find ext no %d on %s", fieldDescriptor.Number(), optionsName) + return false + } + info := f.index.ByDescriptor[extensionField] + err = f.includeElement(info.fullName, extensionField) + return err == nil + }) + return err +} + +func (f *fullNameFilter) includeOptionValue(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) error { + // If the value contains an Any message, we should add the message type + // therein to the closure. + switch { + case fieldDescriptor.IsMap(): + if isMessageKind(fieldDescriptor.MapValue().Kind()) { + var err error + value.Map().Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool { + err = f.includeOptionSingularValueForAny(v.Message()) + return err == nil + }) + return err + } + return nil + case isMessageKind(fieldDescriptor.Kind()): + if fieldDescriptor.IsList() { + listVal := value.List() + for i := 0; i < listVal.Len(); i++ { + if err := f.includeOptionSingularValueForAny(listVal.Get(i).Message()); err != nil { + return err } } + return nil } - filter.include[includeType] = struct{}{} + return f.includeOptionSingularValueForAny(value.Message()) + default: + return nil } - return filter, nil } -func createOptionsFilter(index *imageTypeIndex, options *imageFilterOptions) (fullNameFilter, error) { - var filter fullNameFilter - for includeOption := range options.includeOptions { - includeOption := protoreflect.FullName(includeOption) - if _, ok := index.TypeSet[includeOption]; !ok { - return filter, fmt.Errorf("filtering by included option %q: %w", includeOption, ErrImageFilterTypeNotFound) +func (f *fullNameFilter) includeOptionSingularValueForAny(message protoreflect.Message) error { + md := message.Descriptor() + if md.FullName() == anyFullName { + // Found one! + typeURLFd := md.Fields().ByNumber(1) + if typeURLFd.Kind() != protoreflect.StringKind || typeURLFd.IsList() { + // should not be possible... + return nil } - // TODO: check for imported type filter? - if filter.include == nil { - filter.include = make(map[protoreflect.FullName]struct{}) + typeURL := message.Get(typeURLFd).String() + pos := strings.LastIndexByte(typeURL, '/') + msgType := protoreflect.FullName(typeURL[pos+1:]) + d, _ := f.index.ByName[msgType].element.(*descriptorpb.DescriptorProto) + if d != nil { + if err := f.includeElement(msgType, d); err != nil { + return err + } } - filter.include[includeOption] = struct{}{} + // TODO: unmarshal the bytes to see if there are any nested Any messages + return nil } - for excludeOption := range options.excludeOptions { - excludeOption := protoreflect.FullName(excludeOption) - if _, ok := index.TypeSet[excludeOption]; !ok { - return filter, fmt.Errorf("filtering by excluded option %q: %w", excludeOption, ErrImageFilterTypeNotFound) - } - // TODO: check for imported type filter? - if filter.exclude == nil { - filter.exclude = make(map[protoreflect.FullName]struct{}) + // keep digging + var err error + message.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { + err = f.includeOptionValue(fd, val) + return err == nil + }) + return err +} + +func forEachDescriptor[T namedDescriptor]( + parentName protoreflect.FullName, + list []T, + fn func(protoreflect.FullName, namedDescriptor) error, +) error { + for _, element := range list { + if err := fn(getFullName(parentName, element), element); err != nil { + return err } - filter.exclude[excludeOption] = struct{}{} } - return filter, nil + return nil } -func isDescriptorType(descriptor proto.Message) bool { - switch descriptor := descriptor.(type) { - case *descriptorpb.EnumValueDescriptorProto, *descriptorpb.OneofDescriptorProto: - // Added by their enclosing types. - return false +func isMessageKind(k protoreflect.Kind) bool { + return k == protoreflect.MessageKind || k == protoreflect.GroupKind +} + +func errorUnsupportedFilterType(descriptor namedDescriptor, fullName protoreflect.FullName) error { + var descriptorType string + switch d := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + descriptorType = "file" + case *descriptorpb.DescriptorProto: + descriptorType = "message" case *descriptorpb.FieldDescriptorProto: - return descriptor.Extendee != nil + if d.Extendee != nil { + descriptorType = "extension field" + } else { + descriptorType = "non-extension field" + } + case *descriptorpb.OneofDescriptorProto: + descriptorType = "oneof" + case *descriptorpb.EnumDescriptorProto: + descriptorType = "enum" + case *descriptorpb.EnumValueDescriptorProto: + descriptorType = "enum value" + case *descriptorpb.ServiceDescriptorProto: + descriptorType = "service" + case *descriptorpb.MethodDescriptorProto: + descriptorType = "method" default: - return true + descriptorType = fmt.Sprintf("%T", d) } + return fmt.Errorf("%s is unsupported filter type: %s", fullName, descriptorType) } From 400f9587eb0a71fab2c311b563f72c92d4322c08 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 25 Feb 2025 14:16:45 +0100 Subject: [PATCH 07/80] Fix handling of extensions --- private/buf/bufctl/controller.go | 2 +- private/buf/bufgen/generator.go | 2 +- .../buf/cmd/buf/command/convert/convert.go | 4 +- .../bufimage/bufimageutil/bufimageutil.go | 33 ++--- .../bufimageutil/bufimageutil_test.go | 14 +- .../bufimage/bufimageutil/image_filter.go | 135 +++++++----------- .../bufimage/bufimageutil/image_index.go | 36 ++--- .../bufimage/bufimageutil/image_types.go | 121 ++++++++++------ 8 files changed, 176 insertions(+), 171 deletions(-) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index c57ba10666..617b2e9c31 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -1332,7 +1332,7 @@ func filterImage( newImage = bufimage.ImageWithoutImports(newImage) } if len(functionOptions.imageTypes) > 0 { - newImage, err = bufimageutil.ImageFilteredByTypes(newImage, functionOptions.imageTypes...) + newImage, err = bufimageutil.FilterImage(newImage, bufimageutil.WithIncludeTypes(functionOptions.imageTypes...)) if err != nil { return nil, err } diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 7a2aa5dd92..878a55d083 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -320,7 +320,7 @@ func (g *generator) execLocalPlugin( } if len(excludeOptions) > 0 { for i, pluginImage := range pluginImages { - pluginImage, err := bufimageutil.ExcludeOptions(pluginImage, excludeOptions...) + pluginImage, err := bufimageutil.FilterImage(pluginImage, bufimageutil.WithExcludeOptions(excludeOptions...)) if err != nil { return nil, err } diff --git a/private/buf/cmd/buf/command/convert/convert.go b/private/buf/cmd/buf/command/convert/convert.go index f246f60862..0bd5bf5d94 100644 --- a/private/buf/cmd/buf/command/convert/convert.go +++ b/private/buf/cmd/buf/command/convert/convert.go @@ -185,7 +185,7 @@ func run( resolveWellKnownType = true } if schemaImage != nil { - _, filterErr := bufimageutil.ImageFilteredByTypes(schemaImage, flags.Type) + _, filterErr := bufimageutil.FilterImage(schemaImage, bufimageutil.WithIncludeTypes(flags.Type)) if errors.Is(filterErr, bufimageutil.ErrImageFilterTypeNotFound) { resolveWellKnownType = true } @@ -283,5 +283,5 @@ func wellKnownTypeImage( if err != nil { return nil, err } - return bufimageutil.ImageFilteredByTypes(image, wellKnownTypeName) + return bufimageutil.FilterImage(image, bufimageutil.WithIncludeTypes(wellKnownTypeName)) } diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 6c88624e43..1174968bc8 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -144,14 +144,22 @@ func WithExcludeOptions(typeNames ...string) ImageFilterOption { } } -// ImageFilteredByTypes returns a minimal image containing only the descriptors -// required to define those types. The resulting contains only files in which -// those descriptors and their transitive closure of required descriptors, with -// each file only contains the minimal required types and imports. +// FilterImage returns a minimal image containing only the descriptors +// required to define the set of types provided by the filter options. If no +// filter options are provided, the original image is returned. // -// Although this returns a new [bufimage.Image], it mutates the original image's -// underlying file's [descriptorpb.FileDescriptorProto]. So the old image should -// not continue to be used. +// The filtered image will contain only the files that contain the definitions of +// the specified types, and their transitive dependencies. If a file is no longer +// required, it will be removed from the image. Only the minimal set of types +// required to define the specified types will be included in the filtered image. +// +// Excluded types and options are not included in the filtered image. If an +// included type transitively depens on the excluded type, the descriptor will +// be altered to remove the dependency. +// +// This returns a new [bufimage.Image] that is a shallow copy of the underlying +// [descriptorpb.FileDescriptorProto]s of the original. The new image may therefore +// share state with the original image, so it should not be modified. // // A descriptor is said to require another descriptor if the dependent // descriptor is needed to accurately and completely describe that descriptor. @@ -212,10 +220,6 @@ func WithExcludeOptions(typeNames ...string) ImageFilterOption { // files: [foo.proto, bar.proto] // messages: [pkg.Baz, other.Quux, other.Qux] // extensions: [other.my_option] -func ImageFilteredByTypes(image bufimage.Image, types ...string) (bufimage.Image, error) { - return ImageFilteredByTypesWithOptions(image, types) -} - func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.Image, error) { if len(options) == 0 { return image, nil @@ -227,13 +231,6 @@ func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.I return filterImage(image, filterOptions) } -// ImageFilteredByTypesWithOptions returns a minimal image containing only the descriptors -// required to define those types. See ImageFilteredByTypes for more details. This version -// allows for customizing the behavior with options. -func ImageFilteredByTypesWithOptions(image bufimage.Image, types []string, opts ...ImageFilterOption) (bufimage.Image, error) { - return FilterImage(image, append(opts, WithIncludeTypes(types...))...) -} - // StripSourceRetentionOptions strips any options with a retention of "source" from // the descriptors in the given image. The image is not mutated but instead a new // image is returned. The returned image may share state with the original. diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index fac16862b8..84080ba74e 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -180,7 +180,7 @@ func TestTransitivePublic(t *testing.T) { ) require.NoError(t, err) - filteredImage, err := ImageFilteredByTypes(image, "c.Baz") + filteredImage, err := FilterImage(image, WithIncludeTypes("c.Baz")) require.NoError(t, err) _, err = protodesc.NewFiles(bufimage.ImageToFileDescriptorSet(filteredImage)) @@ -220,15 +220,15 @@ func TestTypesFromMainModule(t *testing.T) { bProtoFileInfo, err := dep.StatFileInfo(ctx, "b.proto") require.NoError(t, err) require.False(t, bProtoFileInfo.IsTargetFile()) - _, err = ImageFilteredByTypes(image, "dependency.Dep") + _, err = FilterImage(image, WithIncludeTypes("dependency.Dep")) require.Error(t, err) assert.ErrorIs(t, err, ErrImageFilterTypeIsImport) // allowed if we specify option - _, err = ImageFilteredByTypesWithOptions(image, []string{"dependency.Dep"}, WithAllowFilterByImportedType()) + _, err = FilterImage(image, WithIncludeTypes("dependency.Dep"), WithAllowFilterByImportedType()) require.NoError(t, err) - _, err = ImageFilteredByTypes(image, "nonexisting") + _, err = FilterImage(image, WithIncludeTypes("nonexisting")) require.Error(t, err) assert.ErrorIs(t, err, ErrImageFilterTypeNotFound) } @@ -259,7 +259,7 @@ func runDiffTest(t *testing.T, testdataDir string, typenames []string, expectedF bucket, image, err := getImage(ctx, slogtestext.NewLogger(t), testdataDir, bufimage.WithExcludeSourceCodeInfo()) require.NoError(t, err) - filteredImage, err := ImageFilteredByTypesWithOptions(image, typenames, opts...) + filteredImage, err := FilterImage(image, append(opts, WithIncludeTypes(typenames...))...) require.NoError(t, err) assert.NotNil(t, image) assert.True(t, imageIsDependencyOrdered(filteredImage), "image files not in dependency order") @@ -323,7 +323,7 @@ func runSourceCodeInfoTest(t *testing.T, typename string, expectedFile string, o bucket, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/sourcecodeinfo") require.NoError(t, err) - filteredImage, err := ImageFilteredByTypesWithOptions(image, []string{typename}, opts...) + filteredImage, err := FilterImage(image, append(opts, WithIncludeTypes(typename))...) require.NoError(t, err) imageFile := filteredImage.GetFile("test.proto") @@ -477,7 +477,7 @@ func benchmarkFilterImage(b *testing.B, opts ...bufimage.BuildImageOption) { require.NoError(b, err) b.StartTimer() - _, err = ImageFilteredByTypes(image, typeName) + _, err = FilterImage(image, WithIncludeTypes(typeName)) require.NoError(b, err) i++ if i == b.N { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index a5c3895f04..fe4432ba86 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -37,7 +37,6 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im if err != nil { return nil, err } - // Loop over image files in revserse DAG order. Imports that are no longer // imported by a previous file are dropped from the image. imageFiles := image.Files() @@ -57,8 +56,6 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im imageFile, imageIndex, filter, - //typeFilter, - //optionsFilter, ) if err != nil { return nil, err @@ -84,8 +81,6 @@ func filterImageFile( imageFile bufimage.ImageFile, imageIndex *imageIndex, filter *fullNameFilter, - //typesFilter fullNameFilter, - //optionsFilter fullNameFilter, ) (bufimage.ImageFile, error) { fileDescriptor := imageFile.FileDescriptorProto() var sourcePathsRemap sourcePathsRemapTrie @@ -132,15 +127,11 @@ func addRemapsForFileDescriptor( fileDescriptor *descriptorpb.FileDescriptorProto, imageIndex *imageIndex, filter *fullNameFilter, - //typesFilter fullNameFilter, - //optionsFilter fullNameFilter, ) (bool, error) { packageName := protoreflect.FullName(fileDescriptor.GetPackage()) if packageName != "" { // Check if filtered by the package name. - isIncluded, isExplicit := filter.hasType(packageName) - if !isIncluded && isExplicit { - // The package is excluded. + if !filter.hasType(packageName) { return false, nil } } @@ -155,18 +146,27 @@ func addRemapsForFileDescriptor( sourcePath := make(protoreflect.SourcePath, 0, 8) // Walk the file descriptor. - if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, builder.addRemapsForDescriptor); err != nil { + isIncluded := false + hasMessages, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, builder.addRemapsForDescriptor) + if err != nil { return false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, builder.addRemapsForEnum); err != nil { + isIncluded = isIncluded || hasMessages + hasEnums, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, builder.addRemapsForEnum) + if err != nil { return false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileServicesTag), fileDescriptor.Service, builder.addRemapsForService); err != nil { + isIncluded = isIncluded || hasEnums + hasServices, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileServicesTag), fileDescriptor.Service, builder.addRemapsForService) + if err != nil { return false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, builder.addRemapsForField); err != nil { + isIncluded = isIncluded || hasServices + hasExtensions, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, builder.addRemapsForField) + if err != nil { return false, err } + isIncluded = isIncluded || hasExtensions if err := builder.addRemapsForOptions(sourcePathsRemap, append(sourcePath, fileOptionsTag), fileDescriptor.Options); err != nil { return false, err } @@ -211,7 +211,7 @@ func addRemapsForFileDescriptor( } } } - return true, nil + return isIncluded, nil } func (b *sourcePathsBuilder) addRemapsForDescriptor( @@ -221,64 +221,44 @@ func (b *sourcePathsBuilder) addRemapsForDescriptor( descriptor *descriptorpb.DescriptorProto, ) (bool, error) { fullName := getFullName(parentName, descriptor) - isIncluded, isExplicit := b.filter.hasType(fullName) - if !isIncluded && isExplicit { + mode := b.filter.inclusionMode(fullName) + if mode == inclusionModeNone { // The type is excluded. return false, nil } - // - // If the message is only enclosin included message remove the fields. - if isIncluded { - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { - return false, err - } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { - return false, err - } - for index, extensionRange := range descriptor.GetExtensionRange() { - extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) - if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { - return false, err - } - } - } else { - sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) - sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) - // TODO: check if extensions are removed??? - sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) - sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) + if mode == inclusionModeEnclosing { + // TODO: check if other descriptor fields are removed? sourcePathsRemap.markDeleted(append(sourcePath, messageReservedRangesTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageReservedNamesTag)) - //for index := range descriptor.GetExtensionRange() { - // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag)) - //} - //sourcePathsRemap.markDeleted(append(sourcePath, messageOptionsTag)) } - // Walk the nested types. - hasNestedTypes, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.addRemapsForDescriptor) - if err != nil { + + // If the message is only enclosing, we search all fields for extensions. + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { return false, err } - isIncluded = isIncluded || hasNestedTypes - - // Walk the enum types. - hasEnums, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.addRemapsForEnum) - if err != nil { + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { return false, err } - isIncluded = isIncluded || hasEnums + for index, extensionRange := range descriptor.GetExtensionRange() { + extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) + if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { + return false, err + } + } - // Walk the oneof types. - hasOneofs, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.addRemapsForOneof) - if err != nil { + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.addRemapsForDescriptor); err != nil { + return false, err + } + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.addRemapsForEnum); err != nil { + return false, err + } + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.addRemapsForOneof); err != nil { return false, err } - isIncluded = isIncluded || hasOneofs - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { return false, err } - return isIncluded, nil + return true, nil } func (b *sourcePathsBuilder) addRemapsForEnum( @@ -289,7 +269,7 @@ func (b *sourcePathsBuilder) addRemapsForEnum( ) (bool, error) { //fullName := b.imageIndex.ByDescriptor[enum] fullName := getFullName(parentName, enum) - if isIncluded, _ := b.filter.hasType(fullName); !isIncluded { + if !b.filter.hasType(fullName) { // The type is excluded, enum values cannot be excluded individually. return false, nil } @@ -316,7 +296,7 @@ func (b *sourcePathsBuilder) addRemapsForOneof( oneof *descriptorpb.OneofDescriptorProto, ) (bool, error) { fullName := getFullName(parentName, oneof) - if isIncluded, _ := b.filter.hasType(fullName); !isIncluded { + if !b.filter.hasType(fullName) { // The type is excluded, enum values cannot be excluded individually. return false, nil } @@ -333,22 +313,18 @@ func (b *sourcePathsBuilder) addRemapsForService( service *descriptorpb.ServiceDescriptorProto, ) (bool, error) { fullName := getFullName(parentName, service) - isIncluded, isExplicit := b.filter.hasType(fullName) - if !isIncluded && isExplicit { + if !b.filter.hasType(fullName) { // The type is excluded. return false, nil } - if isIncluded { - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, serviceOptionsTag), service.GetOptions()); err != nil { - return false, err - } - } // Walk the service methods. - hasMethods, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, serviceMethodsTag), service.Method, b.addRemapsForMethod) - if err != nil { + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, serviceMethodsTag), service.Method, b.addRemapsForMethod); err != nil { return false, err } - return isIncluded || hasMethods, nil + if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, serviceOptionsTag), service.GetOptions()); err != nil { + return false, err + } + return true, nil } func (b *sourcePathsBuilder) addRemapsForMethod( @@ -358,18 +334,18 @@ func (b *sourcePathsBuilder) addRemapsForMethod( method *descriptorpb.MethodDescriptorProto, ) (bool, error) { fullName := getFullName(parentName, method) - if isIncluded, _ := b.filter.hasType(fullName); !isIncluded { + if !b.filter.hasType(fullName) { // The type is excluded. return false, nil } inputName := protoreflect.FullName(strings.TrimPrefix(method.GetInputType(), ".")) - if isIncluded, _ := b.filter.hasType(inputName); !isIncluded { + if !b.filter.hasType(inputName) { // The input type is excluded. return false, fmt.Errorf("input type %s of method %s is excluded", inputName, fullName) } b.addRequiredType(inputName) outputName := protoreflect.FullName(strings.TrimPrefix(method.GetOutputType(), ".")) - if isIncluded, _ := b.filter.hasType(outputName); !isIncluded { + if !b.filter.hasType(outputName) { // The output type is excluded. return false, fmt.Errorf("output type %s of method %s is excluded", outputName, fullName) } @@ -389,17 +365,19 @@ func (b *sourcePathsBuilder) addRemapsForField( if field.Extendee != nil { // This is an extension field. extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) - if isIncluded, _ := b.filter.hasType(extendeeName); !isIncluded { + if !b.filter.hasType(extendeeName) { return false, nil } b.addRequiredType(extendeeName) + } else if b.filter.inclusionMode(parentName) == inclusionModeEnclosing { + return false, nil // The field is excluded. } switch field.GetType() { case descriptorpb.FieldDescriptorProto_TYPE_ENUM, descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, descriptorpb.FieldDescriptorProto_TYPE_GROUP: typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) - if isIncluded, _ := b.filter.hasType(typeName); !isIncluded { + if !b.filter.hasType(typeName) { return false, nil } b.addRequiredType(typeName) @@ -438,9 +416,7 @@ func (b *sourcePathsBuilder) addRemapsForOptions( options := optionsMessage.ProtoReflect() numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - - isIncluded, _ := b.filter.hasOption(fd.FullName(), fd.IsExtension()) - if !isIncluded { + if !b.filter.hasOption(fd.FullName(), fd.IsExtension()) { // Remove this option. optionPath := append(optionsPath, int32(fd.Number())) sourcePathsRemap.markDeleted(optionPath) @@ -627,11 +603,6 @@ func remapListReflect( if fromIndex != int(remapNode.oldIndex) || toIndex != int(remapNode.newIndex) { return fmt.Errorf("unexpected list move %d to %d, expected %d to %d", remapNode.oldIndex, remapNode.newIndex, fromIndex, toIndex) } - //if toIndex != int(remapNode.newIndex) { - // // Mutate the remap node to reflect the actual index. - // // TODO: this is a hack. - // remapNode.newIndex = int32(toIndex) - //} // If no children, the value is unchanged. if len(remapNode.children) > 0 { // Must be a list of messages to have children. diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 575f8b991f..240f8be92a 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -37,15 +37,12 @@ type imageIndex struct { ByName map[protoreflect.FullName]elementInfo // Files maps file names to the file descriptor protos. Files map[string]*descriptorpb.FileDescriptorProto - // NameToExtensions maps fully qualified type names to all known // extension definitions for a type name. NameToExtensions map[string][]*descriptorpb.FieldDescriptorProto - // NameToOptions maps `google.protobuf.*Options` type names to their // known extensions by field tag. NameToOptions map[string]map[int32]*descriptorpb.FieldDescriptorProto - // Packages maps package names to package contents. Packages map[string]*packageInfo } @@ -93,12 +90,28 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* index.NameToExtensions = make(map[string][]*descriptorpb.FieldDescriptorProto) } + addExtension := func(ext *descriptorpb.FieldDescriptorProto) { + extendeeName := strings.TrimPrefix(ext.GetExtendee(), ".") + if options.includeCustomOptions && isOptionsTypeName(extendeeName) { + if _, ok := index.NameToOptions[extendeeName]; !ok { + index.NameToOptions[extendeeName] = make(map[int32]*descriptorpb.FieldDescriptorProto) + } + index.NameToOptions[extendeeName][ext.GetNumber()] = ext + } + if options.includeKnownExtensions { + index.NameToExtensions[extendeeName] = append(index.NameToExtensions[extendeeName], ext) + } + } + for _, imageFile := range image.Files() { pkg := addPackageToIndex(imageFile.FileDescriptorProto().GetPackage(), index) pkg.files = append(pkg.files, imageFile) fileName := imageFile.Path() fileDescriptorProto := imageFile.FileDescriptorProto() index.Files[fileName] = fileDescriptorProto + for _, fd := range fileDescriptorProto.GetExtension() { + addExtension(fd) + } err := walk.DescriptorProtos(fileDescriptorProto, func(name protoreflect.FullName, msg proto.Message) error { if _, existing := index.ByName[name]; existing { return fmt.Errorf("duplicate for %q", name) @@ -137,21 +150,8 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* pkg.types = append(pkg.types, name) } - ext, ok := descriptor.(*descriptorpb.FieldDescriptorProto) - if !ok || ext.Extendee == nil { - // not an extension, so the rest does not apply - return nil - } - - extendeeName := strings.TrimPrefix(ext.GetExtendee(), ".") - if options.includeCustomOptions && isOptionsTypeName(extendeeName) { - if _, ok := index.NameToOptions[extendeeName]; !ok { - index.NameToOptions[extendeeName] = make(map[int32]*descriptorpb.FieldDescriptorProto) - } - index.NameToOptions[extendeeName][ext.GetNumber()] = ext - } - if options.includeKnownExtensions { - index.NameToExtensions[extendeeName] = append(index.NameToExtensions[extendeeName], ext) + if ext, ok := descriptor.(*descriptorpb.FieldDescriptorProto); ok && ext.GetExtendee() != "" { + addExtension(ext) } return nil }) diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go index 80296ccd79..85e8f35c30 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_types.go +++ b/private/bufpkg/bufimage/bufimageutil/image_types.go @@ -23,11 +23,20 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) +type inclusionMode int + +const ( + inclusionModeNone inclusionMode = iota + inclusionModeEnclosing + inclusionModeExplicit +) + type fullNameFilter struct { options *imageFilterOptions index *imageIndex - includes map[protoreflect.FullName]struct{} + includes map[protoreflect.FullName]inclusionMode excludes map[protoreflect.FullName]struct{} + depth int } func newFullNameFilter( @@ -53,38 +62,42 @@ func newFullNameFilter( return nil, err } } + if err := filter.includeExtensions(); err != nil { + return nil, err + } return filter, nil } -func (f *fullNameFilter) hasType(fullName protoreflect.FullName) (isIncluded bool, isExplicit bool) { - if len(f.options.excludeTypes) > 0 || f.excludes != nil { +func (f *fullNameFilter) inclusionMode(fullName protoreflect.FullName) inclusionMode { + if f.excludes != nil { if _, ok := f.excludes[fullName]; ok { - return false, true + return inclusionModeNone } } - if len(f.options.includeTypes) > 0 || f.includes != nil { - if _, ok := f.includes[fullName]; ok { - return true, true - } - return false, false + if f.includes != nil { + return f.includes[fullName] } - return true, false + return inclusionModeExplicit +} + +func (f *fullNameFilter) hasType(fullName protoreflect.FullName) (isIncluded bool) { + return f.inclusionMode(fullName) != inclusionModeNone } -func (f *fullNameFilter) hasOption(fullName protoreflect.FullName, isExtension bool) (isIncluded bool, isExplicit bool) { +func (f *fullNameFilter) hasOption(fullName protoreflect.FullName, isExtension bool) (isIncluded bool) { if f.options.excludeOptions != nil { if _, ok := f.options.excludeOptions[string(fullName)]; ok { - return false, true + return false } } if !f.options.includeCustomOptions { - return !isExtension, true + return !isExtension } if f.options.includeOptions != nil { _, ok := f.options.includeOptions[string(fullName)] - return ok, true + return ok } - return true, false + return true } func (f *fullNameFilter) isExplicitExclude(fullName protoreflect.FullName) bool { @@ -142,7 +155,6 @@ func (f *fullNameFilter) excludeElement(fullName protoreflect.FullName, descript } return nil case *descriptorpb.DescriptorProto: - f.excludes[fullName] = struct{}{} // Exclude all sub-elements if err := forEachDescriptor(fullName, descriptor.GetNestedType(), f.excludeElement); err != nil { return err @@ -176,11 +188,31 @@ func (f *fullNameFilter) include(fullName protoreflect.FullName) error { return nil } if descriptorInfo, ok := f.index.ByName[fullName]; ok { + if err := f.includeElement(fullName, descriptorInfo.element); err != nil { + return err + } // Include the enclosing parent options. - if err := f.includeEnclosingOptions(descriptorInfo.parentName); err != nil { + fileDescriptor := descriptorInfo.imageFile.FileDescriptorProto() + if err := f.includeOptions(fileDescriptor); err != nil { return err } - return f.includeElement(fullName, descriptorInfo.element) + // loop through all enclosing parents since nesting level + // could be arbitrarily deep + for parentName := descriptorInfo.parentName; parentName != ""; { + if isIncluded := f.hasType(parentName); isIncluded { + break + } + f.includes[parentName] = inclusionModeEnclosing + parentInfo, ok := f.index.ByName[parentName] + if !ok { + break + } + if err := f.includeOptions(parentInfo.element); err != nil { + return err + } + parentName = parentInfo.parentName + } + return nil } packageInfo, ok := f.index.Packages[string(fullName)] if !ok { @@ -193,7 +225,6 @@ func (f *fullNameFilter) include(fullName protoreflect.FullName) error { return err } } - f.includes[fullName] = struct{}{} for _, subPackage := range packageInfo.subPackages { if err := f.include(subPackage.fullName); err != nil { return err @@ -210,9 +241,9 @@ func (f *fullNameFilter) includeElement(fullName protoreflect.FullName, descript return nil // already excluded } if f.includes == nil { - f.includes = make(map[protoreflect.FullName]struct{}) + f.includes = make(map[protoreflect.FullName]inclusionMode) } - f.includes[fullName] = struct{}{} + f.includes[fullName] = inclusionModeExplicit if err := f.includeOptions(descriptor); err != nil { return err @@ -243,18 +274,8 @@ func (f *fullNameFilter) includeElement(fullName protoreflect.FullName, descript if err := forEachDescriptor(fullName, descriptor.GetField(), f.includeElement); err != nil { return err } - if err := forEachDescriptor(fullName, descriptor.GetExtension(), f.includeElement); err != nil { - return err - } - - // TODO: Include known extensions. - //if f.options.includeKnownExtensions { - // for _, extensionDescriptor := range f.index.NameToExtensions[string(fullName)] { - // if err := f.includeElement(fullName, extensionDescriptor); err != nil { - // return err - // } - // } - //} + // Extensions are handled after all elements are included. + // This allows us to ensure that the extendee is included first. return nil case *descriptorpb.FieldDescriptorProto: if descriptor.Extendee != nil { @@ -329,19 +350,34 @@ func (f *fullNameFilter) includeElement(fullName protoreflect.FullName, descript } } -func (f *fullNameFilter) includeEnclosingOptions(fullName protoreflect.FullName) error { - // loop through all enclosing parents since nesting level - // could be arbitrarily deep - for info, ok := f.index.ByName[fullName]; ok; { - if err := f.includeOptions(info.element); err != nil { - return err +func (f *fullNameFilter) includeExtensions() error { + if !f.options.includeKnownExtensions { + return nil // nothing to do + } + // TODO + /*if f.options.includeKnownExtensions && len(f.options.includeTypes) > 0 { + for extendee, extensions := range f.index.NameToExtensions { + extendee := protoreflect.FullName(extendee) + if f.inclusionMode(extendee)== inclusionModeNone { + continue + } + for _, extension := range extensions { + typeName := protoreflect.FullName(strings.TrimPrefix(extension.GetTypeName(), ".")) + if typeName == "" && f.hasType(typeName) { + if err := f.includeElement(extendee, extension); err != nil { + return err + } + } + } } - info, ok = f.index.ByName[info.parentName] - } + }*/ return nil } func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { + if !f.options.includeCustomOptions { + return nil + } var optionsMessage proto.Message switch descriptor := descriptor.(type) { case *descriptorpb.FileDescriptorProto: @@ -372,7 +408,7 @@ func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { optionsName := options.Descriptor().FullName() optionsByNumber := f.index.NameToOptions[string(optionsName)] options.Range(func(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) bool { - if isIncluded, _ := f.hasOption(fieldDescriptor.FullName(), fieldDescriptor.IsExtension()); !isIncluded { + if !f.hasOption(fieldDescriptor.FullName(), fieldDescriptor.IsExtension()) { return true } if err = f.includeOptionValue(fieldDescriptor, value); err != nil { @@ -381,6 +417,7 @@ func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { if !fieldDescriptor.IsExtension() { return true } + extensionField, ok := optionsByNumber[int32(fieldDescriptor.Number())] if !ok { err = fmt.Errorf("cannot find ext no %d on %s", fieldDescriptor.Number(), optionsName) From 2a66cf2cd42f5032205841ebe77815fb9ef138b3 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 25 Feb 2025 16:46:00 +0100 Subject: [PATCH 08/80] Fix import types --- .../bufimage/bufimageutil/bufimageutil.go | 13 ++++++++++--- .../bufimage/bufimageutil/image_filter.go | 8 +++++++- .../bufimage/bufimageutil/image_types.go | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 1174968bc8..e74aa4b0b1 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -97,6 +97,8 @@ func WithAllowFilterByImportedType() ImageFilterOption { } } +// WithIncludeTypes returns an option for ImageFilteredByTypesWithOptions that specifies +// the set of types that should be included in the filtered image. func WithIncludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.includeTypes == nil { @@ -108,6 +110,8 @@ func WithIncludeTypes(typeNames ...string) ImageFilterOption { } } +// WithExcludeTypes returns an option for ImageFilteredByTypesWithOptions that specifies +// the set of types that should be excluded from the filtered image. func WithExcludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.excludeTypes == nil { @@ -116,10 +120,11 @@ func WithExcludeTypes(typeNames ...string) ImageFilterOption { for _, typeName := range typeNames { opts.excludeTypes[typeName] = struct{}{} } - //opts.excludeTypes = append(opts.excludeTypes, typeNames...) } } +// WithIncludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies +// the set of options that should be included in the filtered image. func WithIncludeOptions(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.includeOptions == nil { @@ -128,10 +133,13 @@ func WithIncludeOptions(typeNames ...string) ImageFilterOption { for _, typeName := range typeNames { opts.includeOptions[typeName] = struct{}{} } - //opts.includeOptions = append(opts.includeOptions, typeNames...) } } +// WithExcludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies +// the set of options that should be excluded from the filtered image. +// +// May be provided multiple times. func WithExcludeOptions(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.excludeOptions == nil { @@ -140,7 +148,6 @@ func WithExcludeOptions(typeNames ...string) ImageFilterOption { for _, typeName := range typeNames { opts.excludeOptions[typeName] = struct{}{} } - //opts.excludeOptions = append(opts.excludeOptions, typeNames...) } } diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index fe4432ba86..abb85842ba 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -15,6 +15,7 @@ package bufimageutil import ( + "encoding/json" "fmt" "slices" "sort" @@ -37,6 +38,10 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im if err != nil { return nil, err } + b1, _ := json.MarshalIndent(filter.includes, "", " ") + b2, _ := json.MarshalIndent(filter.excludes, "", " ") + fmt.Println("includes", string(b1)) + fmt.Println("excludes", string(b2)) // Loop over image files in revserse DAG order. Imports that are no longer // imported by a previous file are dropped from the image. imageFiles := image.Files() @@ -46,7 +51,7 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im for i := len(image.Files()) - 1; i >= 0; i-- { imageFile := imageFiles[i] imageFilePath := imageFile.Path() - if imageFile.IsImport() { + if imageFile.IsImport() && !options.allowImportedTypes { // Check if this import is still used. if _, isImportUsed := importsByFilePath[imageFilePath]; !isImportUsed { continue @@ -273,6 +278,7 @@ func (b *sourcePathsBuilder) addRemapsForEnum( // The type is excluded, enum values cannot be excluded individually. return false, nil } + fmt.Println("ENUM:", fullName, b.filter.inclusionMode(fullName)) if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, enumOptionsTag), enum.GetOptions()); err != nil { return false, err diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go index 85e8f35c30..8bab5d208a 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_types.go +++ b/private/bufpkg/bufimage/bufimageutil/image_types.go @@ -52,12 +52,18 @@ func newFullNameFilter( } for excludeType := range options.excludeTypes { excludeType := protoreflect.FullName(excludeType) + if err := filter.checkFilterType(excludeType); err != nil { + return nil, err + } if err := filter.exclude(excludeType); err != nil { return nil, err } } for includeType := range options.includeTypes { includeType := protoreflect.FullName(includeType) + if err := filter.checkFilterType(includeType); err != nil { + return nil, err + } if err := filter.include(includeType); err != nil { return nil, err } @@ -81,10 +87,12 @@ func (f *fullNameFilter) inclusionMode(fullName protoreflect.FullName) inclusion } func (f *fullNameFilter) hasType(fullName protoreflect.FullName) (isIncluded bool) { + defer fmt.Println("hasType", fullName, isIncluded) return f.inclusionMode(fullName) != inclusionModeNone } func (f *fullNameFilter) hasOption(fullName protoreflect.FullName, isExtension bool) (isIncluded bool) { + defer fmt.Println("hasOption", fullName, isIncluded) if f.options.excludeOptions != nil { if _, ok := f.options.excludeOptions[string(fullName)]; ok { return false @@ -490,6 +498,17 @@ func (f *fullNameFilter) includeOptionSingularValueForAny(message protoreflect.M return err } +func (f *fullNameFilter) checkFilterType(fullName protoreflect.FullName) error { + info, ok := f.index.ByName[fullName] + if !ok { + return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) + } + if !f.options.allowImportedTypes && info.imageFile.IsImport() { + return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeIsImport) + } + return nil +} + func forEachDescriptor[T namedDescriptor]( parentName protoreflect.FullName, list []T, From 2c4378dc17e91e349540c6dc88e9c542065db3d9 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 26 Feb 2025 00:16:02 +0100 Subject: [PATCH 09/80] Fix extension inclusions --- .../bufimage/bufimageutil/image_filter.go | 119 ++++++++++++++---- .../bufimage/bufimageutil/image_index.go | 7 +- .../bufimage/bufimageutil/image_types.go | 31 ++--- 3 files changed, 113 insertions(+), 44 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index abb85842ba..fcf3b1cb77 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -51,9 +51,11 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im for i := len(image.Files()) - 1; i >= 0; i-- { imageFile := imageFiles[i] imageFilePath := imageFile.Path() + _, isFileImported := importsByFilePath[imageFilePath] if imageFile.IsImport() && !options.allowImportedTypes { // Check if this import is still used. - if _, isImportUsed := importsByFilePath[imageFilePath]; !isImportUsed { + if !isFileImported { + fmt.Println("dropping import", imageFilePath) continue } } @@ -61,12 +63,14 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im imageFile, imageIndex, filter, + isFileImported, ) if err != nil { return nil, err } dirty = dirty || newImageFile != imageFile if newImageFile == nil { + fmt.Println("dropping", imageFilePath) continue // Filtered out. } for _, filePath := range newImageFile.FileDescriptorProto().Dependency { @@ -86,6 +90,7 @@ func filterImageFile( imageFile bufimage.ImageFile, imageIndex *imageIndex, filter *fullNameFilter, + isFileImported bool, ) (bufimage.ImageFile, error) { fileDescriptor := imageFile.FileDescriptorProto() var sourcePathsRemap sourcePathsRemapTrie @@ -98,12 +103,12 @@ func filterImageFile( if err != nil { return nil, err } - if !isIncluded { - return nil, nil // Filtered out. - } if len(sourcePathsRemap) == 0 { return imageFile, nil // No changes required. } + if !isIncluded && !isFileImported { + return nil, nil // Filtered out. + } newFileDescriptor, err := remapFileDescriptor(fileDescriptor, sourcePathsRemap) if err != nil { return nil, err @@ -232,34 +237,36 @@ func (b *sourcePathsBuilder) addRemapsForDescriptor( return false, nil } if mode == inclusionModeEnclosing { - // TODO: check if other descriptor fields are removed? + //fmt.Println("--- ENCLOSING:", fullName) + // If the type is only enclosing, only the namespace matters. + sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageReservedRangesTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageReservedNamesTag)) - } - - // If the message is only enclosing, we search all fields for extensions. - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { - return false, err + } else { + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { + return false, err + } + if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.addRemapsForOneof); err != nil { + return false, err + } + for index, extensionRange := range descriptor.GetExtensionRange() { + extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) + if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { + return false, err + } + } } if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { return false, err } - for index, extensionRange := range descriptor.GetExtensionRange() { - extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) - if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { - return false, err - } - } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.addRemapsForDescriptor); err != nil { return false, err } if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.addRemapsForEnum); err != nil { return false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.addRemapsForOneof); err != nil { - return false, err - } if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { return false, err } @@ -303,7 +310,7 @@ func (b *sourcePathsBuilder) addRemapsForOneof( ) (bool, error) { fullName := getFullName(parentName, oneof) if !b.filter.hasType(fullName) { - // The type is excluded, enum values cannot be excluded individually. + // The type is excluded, oneof fields cannot be excluded individually. return false, nil } if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, oneofOptionsTag), oneof.GetOptions()); err != nil { @@ -369,12 +376,14 @@ func (b *sourcePathsBuilder) addRemapsForField( field *descriptorpb.FieldDescriptorProto, ) (bool, error) { if field.Extendee != nil { - // This is an extension field. - extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) - if !b.filter.hasType(extendeeName) { + // This is an extension field. Extensions are filtered by the name. + extensionFullName := getFullName(parentName, field) + if !b.filter.hasType(extensionFullName) { return false, nil } + extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) b.addRequiredType(extendeeName) + } else if b.filter.inclusionMode(parentName) == inclusionModeEnclosing { return false, nil // The field is excluded. } @@ -419,6 +428,7 @@ func (b *sourcePathsBuilder) addRemapsForOptions( if optionsMessage == nil { return nil } + var err error options := optionsMessage.ProtoReflect() numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { @@ -433,13 +443,70 @@ func (b *sourcePathsBuilder) addRemapsForOptions( // Add the extension type to the required types. b.addRequiredType(fd.FullName()) } - return true + err = b.addOptionValue(fd, val) + return err == nil }) if numFieldsToKeep == 0 { // No options to keep. sourcePathsRemap.markDeleted(optionsPath) } - return nil + return err +} + +func (b *sourcePathsBuilder) addOptionValue( + fieldDescriptor protoreflect.FieldDescriptor, + value protoreflect.Value, +) error { + switch { + case fieldDescriptor.IsMap(): + if isMessageKind(fieldDescriptor.MapValue().Kind()) { + var err error + value.Map().Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool { + err = b.addOptionSingularValueForAny(v.Message()) + return err == nil + }) + return err + } + return nil + case isMessageKind(fieldDescriptor.Kind()): + if fieldDescriptor.IsList() { + listVal := value.List() + for i := 0; i < listVal.Len(); i++ { + if err := b.addOptionSingularValueForAny(listVal.Get(i).Message()); err != nil { + return err + } + } + return nil + } + return b.addOptionSingularValueForAny(value.Message()) + default: + return nil + } +} + +func (b *sourcePathsBuilder) addOptionSingularValueForAny(message protoreflect.Message) error { + md := message.Descriptor() + if md.FullName() == anyFullName { + // Found one! + typeURLFd := md.Fields().ByNumber(1) + if typeURLFd.Kind() != protoreflect.StringKind || typeURLFd.IsList() { + // should not be possible... + return nil + } + typeURL := message.Get(typeURLFd).String() + pos := strings.LastIndexByte(typeURL, '/') + msgType := protoreflect.FullName(typeURL[pos+1:]) + b.addRequiredType(msgType) + // TODO: unmarshal the bytes to see if there are any nested Any messages + return nil + } + // keep digging + var err error + message.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { + err = b.addOptionValue(fd, val) + return err == nil + }) + return err } func (b *sourcePathsBuilder) addRequiredType(fullName protoreflect.FullName) { diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 240f8be92a..6189457a01 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -109,9 +109,9 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* fileName := imageFile.Path() fileDescriptorProto := imageFile.FileDescriptorProto() index.Files[fileName] = fileDescriptorProto - for _, fd := range fileDescriptorProto.GetExtension() { - addExtension(fd) - } + //for _, fd := range fileDescriptorProto.GetExtension() { + // addExtension(fd) + //} err := walk.DescriptorProtos(fileDescriptorProto, func(name protoreflect.FullName, msg proto.Message) error { if _, existing := index.ByName[name]; existing { return fmt.Errorf("duplicate for %q", name) @@ -150,6 +150,7 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* pkg.types = append(pkg.types, name) } + // TODO: needed? if ext, ok := descriptor.(*descriptorpb.FieldDescriptorProto); ok && ext.GetExtendee() != "" { addExtension(ext) } diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go index 8bab5d208a..41b75313be 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_types.go +++ b/private/bufpkg/bufimage/bufimageutil/image_types.go @@ -359,26 +359,28 @@ func (f *fullNameFilter) includeElement(fullName protoreflect.FullName, descript } func (f *fullNameFilter) includeExtensions() error { - if !f.options.includeKnownExtensions { + if !f.options.includeKnownExtensions || len(f.options.includeTypes) == 0 { return nil // nothing to do } - // TODO - /*if f.options.includeKnownExtensions && len(f.options.includeTypes) > 0 { - for extendee, extensions := range f.index.NameToExtensions { - extendee := protoreflect.FullName(extendee) - if f.inclusionMode(extendee)== inclusionModeNone { + for extendeeName, extensions := range f.index.NameToExtensions { + extendeeName := protoreflect.FullName(extendeeName) + if f.inclusionMode(extendeeName) != inclusionModeExplicit { + continue + } + for _, extension := range extensions { + info := f.index.ByDescriptor[extension] + if f.hasType(info.fullName) { continue } - for _, extension := range extensions { - typeName := protoreflect.FullName(strings.TrimPrefix(extension.GetTypeName(), ".")) - if typeName == "" && f.hasType(typeName) { - if err := f.includeElement(extendee, extension); err != nil { - return err - } - } + typeName := protoreflect.FullName(strings.TrimPrefix(extension.GetTypeName(), ".")) + if typeName != "" && !f.hasType(typeName) { + continue + } + if err := f.includeElement(info.fullName, extension); err != nil { + return err } } - }*/ + } return nil } @@ -425,7 +427,6 @@ func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { if !fieldDescriptor.IsExtension() { return true } - extensionField, ok := optionsByNumber[int32(fieldDescriptor.Number())] if !ok { err = fmt.Errorf("cannot find ext no %d on %s", fieldDescriptor.Number(), optionsName) From a0b373ebb803adeecfb12fb7de4e23ecf5b13e4c Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 26 Feb 2025 11:16:49 +0100 Subject: [PATCH 10/80] Checkpoint --- .../bufimage/bufimageutil/bufimageutil.go | 61 ++----------------- .../bufimageutil/bufimageutil_test.go | 11 +++- .../bufimage/bufimageutil/image_filter.go | 22 ++++--- .../bufimage/bufimageutil/image_index.go | 56 +++++++---------- .../bufimage/bufimageutil/image_types.go | 15 ++++- 5 files changed, 62 insertions(+), 103 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index e74aa4b0b1..44217f3040 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -331,11 +331,10 @@ type imageFilterOptions struct { includeCustomOptions bool includeKnownExtensions bool allowImportedTypes bool - - includeTypes map[string]struct{} - excludeTypes map[string]struct{} - includeOptions map[string]struct{} - excludeOptions map[string]struct{} + includeTypes map[string]struct{} + excludeTypes map[string]struct{} + includeOptions map[string]struct{} + excludeOptions map[string]struct{} } func newImageFilterOptions() *imageFilterOptions { @@ -366,55 +365,3 @@ func stripSourceRetentionOptionsFromFile(imageFile bufimage.ImageFile) (bufimage imageFile.UnusedDependencyIndexes(), ) } - -// orderedImports is a structure to maintain an ordered set of imports. This is needed -// because we want to be able to iterate through imports in a deterministic way when filtering -// the image. -type orderedImports struct { - pathToIndex map[string]int - paths []string -} - -// newOrderedImports creates a new orderedImports structure. -func newOrderedImports() *orderedImports { - return &orderedImports{ - pathToIndex: map[string]int{}, - } -} - -// index returns the index for a given path. If the path does not exist in index map, -1 -// is returned and should be considered deleted. -func (o *orderedImports) index(path string) int { - if index, ok := o.pathToIndex[path]; ok { - return index - } - return -1 -} - -// add appends a path to the paths list and the index in the map. If a key already exists, -// then this is a no-op. -func (o *orderedImports) add(path string) { - if _, ok := o.pathToIndex[path]; !ok { - o.pathToIndex[path] = len(o.paths) - o.paths = append(o.paths, path) - } -} - -// delete removes a key from the index map of ordered imports. If a non-existent path is -// set for deletion, then this is a no-op. -// Note that the path is not removed from the paths list. If you want to iterate through -// the paths, use keys() to get all non-deleted keys. -func (o *orderedImports) delete(path string) { - delete(o.pathToIndex, path) -} - -// keys provides all non-deleted keys from the ordered imports. -func (o *orderedImports) keys() []string { - keys := make([]string, 0, len(o.pathToIndex)) - for _, path := range o.paths { - if _, ok := o.pathToIndex[path]; ok { - keys = append(keys, path) - } - } - return keys -} diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 84080ba74e..f347648b99 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -35,6 +35,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/tools/txtar" + "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" @@ -140,7 +141,7 @@ func TestPackages(t *testing.T) { func TestAny(t *testing.T) { t.Parallel() runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax"}, "c1.txtar") - runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InField"}, "c2.txtar") + runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InField"}, "c2.txtar") // c2 runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InList"}, "c3.txtar") runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InMap"}, "c4.txtar") runDiffTest(t, "testdata/any", []string{"NormalMessageSyntaxValidType"}, "d.txtar") @@ -259,6 +260,10 @@ func runDiffTest(t *testing.T, testdataDir string, typenames []string, expectedF bucket, image, err := getImage(ctx, slogtestext.NewLogger(t), testdataDir, bufimage.WithExcludeSourceCodeInfo()) require.NoError(t, err) + _ = protojson.MarshalOptions{} + //b, _ := protojson.MarshalOptions{Indent: " "}.Marshal(bufimage.ImageToFileDescriptorSet(image)) + //t.Log(string(b)) + filteredImage, err := FilterImage(image, append(opts, WithIncludeTypes(typenames...))...) require.NoError(t, err) assert.NotNil(t, image) @@ -275,6 +280,10 @@ func runDiffTest(t *testing.T, testdataDir string, typenames []string, expectedF err = protoencoding.NewWireUnmarshaler(filteredImage.Resolver()).Unmarshal(data, fileDescriptorSet) require.NoError(t, err) + t.Log("--------------------------------------------") + //b, _ = protojson.MarshalOptions{Indent: " "}.Marshal(bufimage.ImageToFileDescriptorSet(image)) + //t.Log(string(b)) + files, err := protodesc.NewFiles(fileDescriptorSet) require.NoError(t, err) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index fcf3b1cb77..89bed7aa0d 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -34,10 +34,15 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im if err != nil { return nil, err } + for SOMETHING, pkg := range imageIndex.Packages { + fmt.Println("PACKAGE", SOMETHING, pkg.fullName) + } filter, err := newFullNameFilter(imageIndex, options) if err != nil { + fmt.Println("WAT", err) return nil, err } + fmt.Println("----") b1, _ := json.MarshalIndent(filter.includes, "", " ") b2, _ := json.MarshalIndent(filter.excludes, "", " ") fmt.Println("includes", string(b1)) @@ -55,7 +60,6 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im if imageFile.IsImport() && !options.allowImportedTypes { // Check if this import is still used. if !isFileImported { - fmt.Println("dropping import", imageFilePath) continue } } @@ -70,7 +74,6 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im } dirty = dirty || newImageFile != imageFile if newImageFile == nil { - fmt.Println("dropping", imageFilePath) continue // Filtered out. } for _, filePath := range newImageFile.FileDescriptorProto().Dependency { @@ -103,12 +106,12 @@ func filterImageFile( if err != nil { return nil, err } - if len(sourcePathsRemap) == 0 { - return imageFile, nil // No changes required. - } if !isIncluded && !isFileImported { return nil, nil // Filtered out. } + if len(sourcePathsRemap) == 0 { + return imageFile, nil // No changes required. + } newFileDescriptor, err := remapFileDescriptor(fileDescriptor, sourcePathsRemap) if err != nil { return nil, err @@ -138,10 +141,12 @@ func addRemapsForFileDescriptor( imageIndex *imageIndex, filter *fullNameFilter, ) (bool, error) { + fmt.Println("FILE", fileDescriptor.GetName()) packageName := protoreflect.FullName(fileDescriptor.GetPackage()) if packageName != "" { // Check if filtered by the package name. if !filter.hasType(packageName) { + fmt.Println("FILTERED BY PACKAGE", packageName) return false, nil } } @@ -237,7 +242,7 @@ func (b *sourcePathsBuilder) addRemapsForDescriptor( return false, nil } if mode == inclusionModeEnclosing { - //fmt.Println("--- ENCLOSING:", fullName) + fmt.Println("--- ENCLOSING:", fullName) // If the type is only enclosing, only the namespace matters. sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) @@ -285,8 +290,6 @@ func (b *sourcePathsBuilder) addRemapsForEnum( // The type is excluded, enum values cannot be excluded individually. return false, nil } - fmt.Println("ENUM:", fullName, b.filter.inclusionMode(fullName)) - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, enumOptionsTag), enum.GetOptions()); err != nil { return false, err } @@ -496,6 +499,9 @@ func (b *sourcePathsBuilder) addOptionSingularValueForAny(message protoreflect.M typeURL := message.Get(typeURLFd).String() pos := strings.LastIndexByte(typeURL, '/') msgType := protoreflect.FullName(typeURL[pos+1:]) + if !b.filter.hasType(msgType) { + return nil + } b.addRequiredType(msgType) // TODO: unmarshal the bytes to see if there are any nested Any messages return nil diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 6189457a01..ea531f902a 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -90,28 +90,12 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* index.NameToExtensions = make(map[string][]*descriptorpb.FieldDescriptorProto) } - addExtension := func(ext *descriptorpb.FieldDescriptorProto) { - extendeeName := strings.TrimPrefix(ext.GetExtendee(), ".") - if options.includeCustomOptions && isOptionsTypeName(extendeeName) { - if _, ok := index.NameToOptions[extendeeName]; !ok { - index.NameToOptions[extendeeName] = make(map[int32]*descriptorpb.FieldDescriptorProto) - } - index.NameToOptions[extendeeName][ext.GetNumber()] = ext - } - if options.includeKnownExtensions { - index.NameToExtensions[extendeeName] = append(index.NameToExtensions[extendeeName], ext) - } - } - for _, imageFile := range image.Files() { pkg := addPackageToIndex(imageFile.FileDescriptorProto().GetPackage(), index) pkg.files = append(pkg.files, imageFile) fileName := imageFile.Path() fileDescriptorProto := imageFile.FileDescriptorProto() index.Files[fileName] = fileDescriptorProto - //for _, fd := range fileDescriptorProto.GetExtension() { - // addExtension(fd) - //} err := walk.DescriptorProtos(fileDescriptorProto, func(name protoreflect.FullName, msg proto.Message) error { if _, existing := index.ByName[name]; existing { return fmt.Errorf("duplicate for %q", name) @@ -127,33 +111,37 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* // certain descriptor types don't need to be indexed: // enum values, normal (non-extension) fields, and oneofs - var includeInIndex bool switch d := descriptor.(type) { case *descriptorpb.EnumValueDescriptorProto, *descriptorpb.OneofDescriptorProto: // do not add to package elements; these elements are implicitly included by their enclosing type + return nil case *descriptorpb.FieldDescriptorProto: // only add to elements if an extension (regular fields implicitly included by containing message) - includeInIndex = d.Extendee != nil - default: - includeInIndex = true - } - - if includeInIndex { - info := elementInfo{ - fullName: name, - imageFile: imageFile, - parentName: parentName, - element: descriptor, + if d.Extendee == nil { + return nil + } + extendeeName := strings.TrimPrefix(d.GetExtendee(), ".") + if options.includeCustomOptions && isOptionsTypeName(extendeeName) { + if _, ok := index.NameToOptions[extendeeName]; !ok { + index.NameToOptions[extendeeName] = make(map[int32]*descriptorpb.FieldDescriptorProto) + } + index.NameToOptions[extendeeName][d.GetNumber()] = d + } + if options.includeKnownExtensions { + index.NameToExtensions[extendeeName] = append(index.NameToExtensions[extendeeName], d) } - index.ByName[name] = info - index.ByDescriptor[descriptor] = info - pkg.types = append(pkg.types, name) } - // TODO: needed? - if ext, ok := descriptor.(*descriptorpb.FieldDescriptorProto); ok && ext.GetExtendee() != "" { - addExtension(ext) + info := elementInfo{ + fullName: name, + imageFile: imageFile, + parentName: parentName, + element: descriptor, } + index.ByName[name] = info + index.ByDescriptor[descriptor] = info + pkg.types = append(pkg.types, name) + return nil }) if err != nil { diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go index 41b75313be..648773a135 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_types.go +++ b/private/bufpkg/bufimage/bufimageutil/image_types.go @@ -117,6 +117,7 @@ func (f *fullNameFilter) isExplicitExclude(fullName protoreflect.FullName) bool } func (f *fullNameFilter) exclude(fullName protoreflect.FullName) error { + fmt.Println("exclude", fullName) if _, excluded := f.excludes[fullName]; excluded { return nil } @@ -192,6 +193,7 @@ func (f *fullNameFilter) excludeElement(fullName protoreflect.FullName, descript } func (f *fullNameFilter) include(fullName protoreflect.FullName) error { + fmt.Println("include", fullName) if _, included := f.includes[fullName]; included { return nil } @@ -385,6 +387,7 @@ func (f *fullNameFilter) includeExtensions() error { } func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { + fmt.Println("includeOptions", descriptor.ProtoReflect().Descriptor().FullName()) if !f.options.includeCustomOptions { return nil } @@ -500,11 +503,17 @@ func (f *fullNameFilter) includeOptionSingularValueForAny(message protoreflect.M } func (f *fullNameFilter) checkFilterType(fullName protoreflect.FullName) error { - info, ok := f.index.ByName[fullName] - if !ok { + var isImport bool + if info, ok := f.index.ByName[fullName]; ok { + isImport = info.imageFile.IsImport() + } else if pkg, ok := f.index.Packages[string(fullName)]; ok { + for _, file := range pkg.files { + isImport = isImport || file.IsImport() + } + } else { return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) } - if !f.options.allowImportedTypes && info.imageFile.IsImport() { + if !f.options.allowImportedTypes && isImport { return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeIsImport) } return nil From 11a88a939d1378786a4b046f38ae28297f4082bd Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 14:23:23 +0100 Subject: [PATCH 11/80] Refactor for transitive closure --- .golangci.yml | 2 +- .../bufimage/bufimageutil/bufimageutil.go | 600 ++++++++++++ .../bufimage/bufimageutil/image_filter.go | 852 ++++++++---------- .../bufimageutil/image_filter_test.go | 4 +- .../bufimage/bufimageutil/image_index.go | 53 +- .../bufimage/bufimageutil/image_types.go | 566 ------------ 6 files changed, 1028 insertions(+), 1049 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimageutil/image_types.go diff --git a/.golangci.yml b/.golangci.yml index c25e0bec9b..a57119ad1e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -367,7 +367,7 @@ issues: - gosec # bufimageutil is handling images and converting loop indices to int32. Since it is # parsing from an Image, the operation should be safe. - path: private/bufpkg/bufimage/bufimageutil/exclude_options.go + path: private/bufpkg/bufimage/bufimageutil/image_filter.go text: "G115:" - linters: - gosec diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 44217f3040..0fa8e24412 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -23,6 +23,8 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/protoplugin/protopluginutil" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) @@ -253,6 +255,552 @@ func StripSourceRetentionOptions(image bufimage.Image) (bufimage.Image, error) { return bufimage.NewImage(updatedFiles) } +// transitiveClosure accumulates the elements, files, and needed imports for a +// subset of an image. When an element is added to the closure, all of its +// dependencies are recursively added. +type transitiveClosure struct { + // The elements included in the transitive closure. + elements map[namedDescriptor]closureInclusionMode + // The ordered set of imports for each file. This allows for re-writing imports + // for files whose contents have been pruned. + imports map[string]*orderedImports +} + +type closureInclusionMode int + +const ( + // Element is explicitly excluded from the closure. + inclusionModeExcluded = closureInclusionMode(iota - 1) // -1 + // Element is not yet known to be included or excluded. + inclusionModeUnknown // 0 + // Element is included in closure because it is directly reachable from a root. + inclusionModeExplicit // 1 + // Element is included in closure because it is a message or service that + // *contains* an explicitly included element but is not itself directly + // reachable. + inclusionModeEnclosing // 2 + // Element is included in closure because it is implied by the presence of a + // custom option. For example, a field element with a custom option implies + // the presence of google.protobuf.FieldOptions. An option type could instead be + // explicitly included if it is also directly reachable (i.e. some type in the + // graph explicitly refers to the option type). + inclusionModeImplicit // 3 +) + +func newTransitiveClosure() *transitiveClosure { + return &transitiveClosure{ + elements: map[namedDescriptor]closureInclusionMode{}, + imports: map[string]*orderedImports{}, + } +} + +func (t *transitiveClosure) hasType( + descriptor namedDescriptor, + options *imageFilterOptions, +) (isIncluded bool) { + defer func() { fmt.Println("\thasType", descriptor.GetName(), isIncluded) }() + if options == nil { + return true // no filter + } + switch mode := t.elements[descriptor]; mode { + case inclusionModeExplicit, inclusionModeImplicit, inclusionModeEnclosing: + return true + case inclusionModeExcluded: + return false + case inclusionModeUnknown: + // True if no includes are specified. + return options.includeTypes == nil + default: + return false + } +} + +func (t *transitiveClosure) hasOption( + fieldDescriptor protoreflect.FieldDescriptor, + options *imageFilterOptions, +) (isIncluded bool) { + defer func() { fmt.Println("\thasOption", fieldDescriptor.FullName(), isIncluded) }() + fullName := fieldDescriptor.FullName() + defer fmt.Println("\thasOption", fullName, isIncluded) + if options == nil { + return true // no filter + } + if options.excludeTypes != nil { + if _, ok := options.excludeTypes[string(fullName)]; ok { + return false + } + } + if fieldDescriptor.IsExtension() && !options.includeCustomOptions { + return false + } + if options.includeOptions != nil { + _, isIncluded = options.includeOptions[string(fullName)] + return isIncluded + } + return true +} + +func (t *transitiveClosure) excludeElement( + descriptor namedDescriptor, + imageIndex *imageIndex, + options *imageFilterOptions, +) error { + _ = descriptor + // TODO: implement + return nil +} + +func (t *transitiveClosure) addType( + typeName protoreflect.FullName, + imageIndex *imageIndex, + options *imageFilterOptions, +) error { + // TODO: consider supporting a glob syntax of some kind, to do more advanced pattern + // matching, such as ability to get a package AND all of its sub-packages. + descriptorInfo, ok := imageIndex.ByName[typeName] + if ok { + // It's a type name + if !options.allowImportedTypes && descriptorInfo.file.IsImport() { + return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + } + return t.addElement(descriptorInfo.element, "", false, imageIndex, options) + } + // It could be a package name + pkg, ok := imageIndex.Packages[string(typeName)] + if !ok { + // but it's not... + return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeNotFound) + } + if !options.allowImportedTypes { + // if package includes only imported files, then reject + onlyImported := true + for _, file := range pkg.files { + if !file.IsImport() { + onlyImported = false + break + } + } + if onlyImported { + return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + } + } + return t.addPackage(pkg, imageIndex, options) +} + +func (t *transitiveClosure) addImport(fromPath, toPath string) { + if fromPath == toPath { + return // no need for a file to import itself + } + imps := t.imports[fromPath] + if imps == nil { + imps = newOrderedImports() + t.imports[fromPath] = imps + } + imps.add(toPath) +} + +func (t *transitiveClosure) addPackage( + pkg *packageInfo, + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + fmt.Printf("ADD PACKAGE: %q\n", pkg.fullName) + for _, file := range pkg.files { + fmt.Println("\taddPackage", file.Path()) + fileDescriptor := file.FileDescriptorProto() + if err := t.addElement(fileDescriptor, "", false, imageIndex, opts); err != nil { + return err + } + } + return nil +} + +func (t *transitiveClosure) addElement( + descriptor namedDescriptor, + referrerFile string, + impliedByCustomOption bool, + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + descriptorInfo := imageIndex.ByDescriptor[descriptor] + if referrerFile != "" { + t.addImport(referrerFile, descriptorInfo.file.Path()) + } + + if existingMode, ok := t.elements[descriptor]; ok && existingMode != inclusionModeEnclosing { + if existingMode == inclusionModeImplicit && !impliedByCustomOption { + // upgrade from implied to explicitly part of closure + t.elements[descriptor] = inclusionModeExplicit + } + return nil // already added this element + } + if impliedByCustomOption { + t.elements[descriptor] = inclusionModeImplicit + } else { + t.elements[descriptor] = inclusionModeExplicit + } + + // if this type is enclosed inside another, add enclosing types + fmt.Println("--- ADDING ELEMENT", descriptorInfo.fullName, "=>", t.elements[descriptor]) + fmt.Printf("\t %s %T\n", descriptorInfo.file.Path(), descriptorInfo.parent) + if err := t.addEnclosing(descriptorInfo.parent, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + // add any custom options and their dependencies + if err := t.exploreCustomOptions(descriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + + switch typedDescriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + typeNames, ok := imageIndex.FileTypes[typedDescriptor.GetName()] + if !ok { + return fmt.Errorf("missing %q", typedDescriptor.GetName()) + } + // A file includes all elements. The types are resolved in the image index + // to ensure all nested types are included. + for _, typeName := range typeNames { + typeInfo := imageIndex.ByName[typeName] + if err := t.addElement(typeInfo.element, "", false, imageIndex, opts); err != nil { + return err + } + } + + case *descriptorpb.DescriptorProto: + // Options and types for all fields + for _, field := range typedDescriptor.GetField() { + if err := t.addFieldType(field, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + if err := t.exploreCustomOptions(field, referrerFile, imageIndex, opts); err != nil { + return err + } + } + // Options for all oneofs in this message + for _, oneOfDescriptor := range typedDescriptor.GetOneofDecl() { + if err := t.exploreCustomOptions(oneOfDescriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + } + // Options for all extension ranges in this message + for _, extRange := range typedDescriptor.GetExtensionRange() { + if err := t.exploreCustomOptions(extRange, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + } + + case *descriptorpb.EnumDescriptorProto: + for _, enumValue := range typedDescriptor.GetValue() { + if err := t.exploreCustomOptions(enumValue, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + } + + case *descriptorpb.ServiceDescriptorProto: + for _, method := range typedDescriptor.GetMethod() { + if err := t.addElement(method, "", false, imageIndex, opts); err != nil { + return err + } + } + + case *descriptorpb.MethodDescriptorProto: + inputName := protoreflect.FullName(strings.TrimPrefix(typedDescriptor.GetInputType(), ".")) + inputInfo, ok := imageIndex.ByName[inputName] + if !ok { + return fmt.Errorf("missing %q", inputName) + } + if err := t.addElement(inputInfo.element, descriptorInfo.file.Path(), false, imageIndex, opts); err != nil { + return err + } + + outputName := protoreflect.FullName(strings.TrimPrefix(typedDescriptor.GetOutputType(), ".")) + outputInfo, ok := imageIndex.ByName[outputName] + if !ok { + return fmt.Errorf("missing %q", outputName) + } + if err := t.addElement(outputInfo.element, descriptorInfo.file.Path(), false, imageIndex, opts); err != nil { + return err + } + + case *descriptorpb.FieldDescriptorProto: + fmt.Println("ADDING EXTENSION", typedDescriptor.GetExtendee()) + // Regular fields are handled above in message descriptor case. + // We should only find our way here for extensions. + if typedDescriptor.Extendee == nil { + return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) + } + if typedDescriptor.GetExtendee() == "" { + return fmt.Errorf("expected extendee for field %q to not be empty", descriptorInfo.fullName) + } + extendeeName := protoreflect.FullName(strings.TrimPrefix(typedDescriptor.GetExtendee(), ".")) + extendeeInfo, ok := imageIndex.ByName[extendeeName] + if !ok { + return fmt.Errorf("missing %q", extendeeName) + } + if err := t.addElement(extendeeInfo.element, descriptorInfo.file.Path(), impliedByCustomOption, imageIndex, opts); err != nil { + return err + } + if err := t.addFieldType(typedDescriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + + default: + return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) + } + + return nil +} + +func errorUnsupportedFilterType(descriptor namedDescriptor, fullName protoreflect.FullName) error { + var descriptorType string + switch d := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + descriptorType = "file" + case *descriptorpb.DescriptorProto: + descriptorType = "message" + case *descriptorpb.FieldDescriptorProto: + if d.Extendee != nil { + descriptorType = "extension field" + } else { + descriptorType = "non-extension field" + } + case *descriptorpb.OneofDescriptorProto: + descriptorType = "oneof" + case *descriptorpb.EnumDescriptorProto: + descriptorType = "enum" + case *descriptorpb.EnumValueDescriptorProto: + descriptorType = "enum value" + case *descriptorpb.ServiceDescriptorProto: + descriptorType = "service" + case *descriptorpb.MethodDescriptorProto: + descriptorType = "method" + default: + descriptorType = fmt.Sprintf("%T", d) + } + return fmt.Errorf("%s is unsupported filter type: %s", fullName, descriptorType) +} + +func (t *transitiveClosure) addEnclosing(descriptor namedDescriptor, enclosingFile string, imageIndex *imageIndex, opts *imageFilterOptions) error { + // loop through all enclosing parents since nesting level + // could be arbitrarily deep + for descriptor != nil { + _, isMsg := descriptor.(*descriptorpb.DescriptorProto) + _, isSvc := descriptor.(*descriptorpb.ServiceDescriptorProto) + _, isFile := descriptor.(*descriptorpb.FileDescriptorProto) + if !isMsg && !isSvc && !isFile { + break // not an enclosing type + } + if _, ok := t.elements[descriptor]; ok { + break // already in closure + } + t.elements[descriptor] = inclusionModeEnclosing + if err := t.exploreCustomOptions(descriptor, enclosingFile, imageIndex, opts); err != nil { + return err + } + // now move into this element's parent + descriptor = imageIndex.ByDescriptor[descriptor].parent + } + return nil +} + +func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProto, referrerFile string, imageIndex *imageIndex, opts *imageFilterOptions) error { + switch field.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_ENUM, + descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, + descriptorpb.FieldDescriptorProto_TYPE_GROUP: + typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) + info, ok := imageIndex.ByName[typeName] + if !ok { + return fmt.Errorf("missing %q", typeName) + } + err := t.addElement(info.element, referrerFile, false, imageIndex, opts) + if err != nil { + return err + } + case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, + descriptorpb.FieldDescriptorProto_TYPE_FLOAT, + descriptorpb.FieldDescriptorProto_TYPE_INT64, + descriptorpb.FieldDescriptorProto_TYPE_UINT64, + descriptorpb.FieldDescriptorProto_TYPE_INT32, + descriptorpb.FieldDescriptorProto_TYPE_FIXED64, + descriptorpb.FieldDescriptorProto_TYPE_FIXED32, + descriptorpb.FieldDescriptorProto_TYPE_BOOL, + descriptorpb.FieldDescriptorProto_TYPE_STRING, + descriptorpb.FieldDescriptorProto_TYPE_BYTES, + descriptorpb.FieldDescriptorProto_TYPE_UINT32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, + descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, + descriptorpb.FieldDescriptorProto_TYPE_SINT32, + descriptorpb.FieldDescriptorProto_TYPE_SINT64: + // nothing to follow, custom options handled below. + default: + return fmt.Errorf("unknown field type %d", field.GetType()) + } + return nil +} + +func (t *transitiveClosure) addExtensions( + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + if !opts.includeKnownExtensions { + return nil // nothing to do + } + for e, mode := range t.elements { + if mode != inclusionModeExplicit { + // we only collect extensions for messages that are directly reachable/referenced. + continue + } + fmt.Println("ADDING EXTENSIONS FOR", e.GetName()) + msgDescriptor, ok := e.(*descriptorpb.DescriptorProto) + if !ok { + // not a message, nothing to do + continue + } + fmt.Println("\t", msgDescriptor.GetName()) + descriptorInfo := imageIndex.ByDescriptor[msgDescriptor] + for _, extendsDescriptor := range imageIndex.NameToExtensions[descriptorInfo.fullName] { + fmt.Println("\t\t", extendsDescriptor.GetName()) + if err := t.addElement(extendsDescriptor, "", false, imageIndex, opts); err != nil { + return err + } + } + } + return nil +} + +func (t *transitiveClosure) exploreCustomOptions( + descriptor proto.Message, + referrerFile string, + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + if !opts.includeCustomOptions { + return nil + } + + var options protoreflect.Message + switch descriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.DescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.FieldDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.OneofDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.EnumDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.EnumValueDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.ServiceDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.MethodDescriptorProto: + options = descriptor.GetOptions().ProtoReflect() + case *descriptorpb.DescriptorProto_ExtensionRange: + options = descriptor.GetOptions().ProtoReflect() + default: + return fmt.Errorf("unexpected type for exploring options %T", descriptor) + } + + optionsName := options.Descriptor().FullName() + var err error + options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { + // If the value contains an Any message, we should add the message type + // therein to the closure. + if err = t.exploreOptionValueForAny(fd, val, referrerFile, imageIndex, opts); err != nil { + return false + } + + // Also include custom option definitions (e.g. extensions) + if !fd.IsExtension() { + return true + } + optionsByNumber := imageIndex.NameToOptions[optionsName] + field, ok := optionsByNumber[int32(fd.Number())] + if !ok { + err = fmt.Errorf("cannot find ext no %d on %s", fd.Number(), optionsName) + return false + } + err = t.addElement(field, referrerFile, true, imageIndex, opts) + return err == nil + }) + return err +} + +func isMessageKind(k protoreflect.Kind) bool { + return k == protoreflect.MessageKind || k == protoreflect.GroupKind +} + +func (t *transitiveClosure) exploreOptionValueForAny( + fd protoreflect.FieldDescriptor, + val protoreflect.Value, + referrerFile string, + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + switch { + case fd.IsMap(): + if isMessageKind(fd.MapValue().Kind()) { + var err error + val.Map().Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool { + if err = t.exploreOptionSingularValueForAny(v.Message(), referrerFile, imageIndex, opts); err != nil { + return false + } + return true + }) + return err + } + case isMessageKind(fd.Kind()): + if fd.IsList() { + listVal := val.List() + for i := 0; i < listVal.Len(); i++ { + if err := t.exploreOptionSingularValueForAny(listVal.Get(i).Message(), referrerFile, imageIndex, opts); err != nil { + return err + } + } + } else { + return t.exploreOptionSingularValueForAny(val.Message(), referrerFile, imageIndex, opts) + } + } + return nil +} + +func (t *transitiveClosure) exploreOptionSingularValueForAny( + msg protoreflect.Message, + referrerFile string, + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + md := msg.Descriptor() + if md.FullName() == anyFullName { + // Found one! + typeURLFd := md.Fields().ByNumber(1) + if typeURLFd.Kind() != protoreflect.StringKind || typeURLFd.IsList() { + // should not be possible... + return nil + } + typeURL := msg.Get(typeURLFd).String() + pos := strings.LastIndexByte(typeURL, '/') + msgType := protoreflect.FullName(typeURL[pos+1:]) + d, _ := imageIndex.ByName[msgType].element.(*descriptorpb.DescriptorProto) + if d != nil { + if err := t.addElement(d, referrerFile, false, imageIndex, opts); err != nil { + return err + } + } + // TODO: unmarshal the bytes to see if there are any nested Any messages + return nil + } + // keep digging + var err error + msg.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { + err = t.exploreOptionValueForAny(fd, val, referrerFile, imageIndex, opts) + return err == nil + }) + return err +} + type int32Range struct { start, end int32 // both inclusive } @@ -365,3 +913,55 @@ func stripSourceRetentionOptionsFromFile(imageFile bufimage.ImageFile) (bufimage imageFile.UnusedDependencyIndexes(), ) } + +// orderedImports is a structure to maintain an ordered set of imports. This is needed +// because we want to be able to iterate through imports in a deterministic way when filtering +// the image. +type orderedImports struct { + pathToIndex map[string]int + paths []string +} + +// newOrderedImports creates a new orderedImports structure. +func newOrderedImports() *orderedImports { + return &orderedImports{ + pathToIndex: map[string]int{}, + } +} + +// index returns the index for a given path. If the path does not exist in index map, -1 +// is returned and should be considered deleted. +func (o *orderedImports) index(path string) int { + if index, ok := o.pathToIndex[path]; ok { + return index + } + return -1 +} + +// add appends a path to the paths list and the index in the map. If a key already exists, +// then this is a no-op. +func (o *orderedImports) add(path string) { + if _, ok := o.pathToIndex[path]; !ok { + o.pathToIndex[path] = len(o.paths) + o.paths = append(o.paths, path) + } +} + +// delete removes a key from the index map of ordered imports. If a non-existent path is +// set for deletion, then this is a no-op. +// Note that the path is not removed from the paths list. If you want to iterate through +// the paths, use keys() to get all non-deleted keys. +func (o *orderedImports) delete(path string) { + delete(o.pathToIndex, path) +} + +// keys provides all non-deleted keys from the ordered imports. +func (o *orderedImports) keys() []string { + keys := make([]string, 0, len(o.pathToIndex)) + for _, path := range o.paths { + if _, ok := o.pathToIndex[path]; ok { + keys = append(keys, path) + } + } + return keys +} diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 89bed7aa0d..2ee645f6a7 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -15,14 +15,11 @@ package bufimageutil import ( - "encoding/json" "fmt" "slices" - "sort" "strings" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/syserror" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" @@ -37,16 +34,23 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im for SOMETHING, pkg := range imageIndex.Packages { fmt.Println("PACKAGE", SOMETHING, pkg.fullName) } - filter, err := newFullNameFilter(imageIndex, options) - if err != nil { - fmt.Println("WAT", err) + closure := newTransitiveClosure() + // TODO: handle options. + // TODO: add all excludes first. + for includeType := range options.includeTypes { + includeType := protoreflect.FullName(includeType) + if err := closure.addType(includeType, imageIndex, options); err != nil { + return nil, err + } + } + // After all types are added, add their known extensions + if err := closure.addExtensions(imageIndex, options); err != nil { return nil, err } - fmt.Println("----") - b1, _ := json.MarshalIndent(filter.includes, "", " ") - b2, _ := json.MarshalIndent(filter.excludes, "", " ") - fmt.Println("includes", string(b1)) - fmt.Println("excludes", string(b2)) + for key, value := range closure.elements { + fmt.Println("CLOSURE", key.GetName(), value) + } + // Loop over image files in revserse DAG order. Imports that are no longer // imported by a previous file are dropped from the image. imageFiles := image.Files() @@ -56,24 +60,30 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im for i := len(image.Files()) - 1; i >= 0; i-- { imageFile := imageFiles[i] imageFilePath := imageFile.Path() + fmt.Println("IMAGE FILE", i, imageFilePath) _, isFileImported := importsByFilePath[imageFilePath] if imageFile.IsImport() && !options.allowImportedTypes { // Check if this import is still used. if !isFileImported { + fmt.Println(" NOT IMPORTED") continue } } newImageFile, err := filterImageFile( imageFile, imageIndex, - filter, - isFileImported, + closure, + options, ) if err != nil { return nil, err } dirty = dirty || newImageFile != imageFile if newImageFile == nil { + if isFileImported { + return nil, fmt.Errorf("imported file %q was filtered out", imageFilePath) + } + fmt.Println(" FILTERED OUT") continue // Filtered out. } for _, filePath := range newImageFile.FileDescriptorProto().Dependency { @@ -82,6 +92,7 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im newImageFiles = append(newImageFiles, newImageFile) } if dirty { + fmt.Println("Creating the new image") // Reverse the image files back to DAG order. slices.Reverse(newImageFiles) return bufimage.NewImage(newImageFiles) @@ -92,29 +103,59 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im func filterImageFile( imageFile bufimage.ImageFile, imageIndex *imageIndex, - filter *fullNameFilter, - isFileImported bool, + closure *transitiveClosure, + options *imageFilterOptions, ) (bufimage.ImageFile, error) { fileDescriptor := imageFile.FileDescriptorProto() var sourcePathsRemap sourcePathsRemapTrie - isIncluded, err := addRemapsForFileDescriptor( - &sourcePathsRemap, - fileDescriptor, - imageIndex, - filter, - ) + builder := sourcePathsBuilder{ + imageIndex: imageIndex, + closure: closure, + options: options, + } + newFileDescriptor, err := builder.remapFileDescriptor(&sourcePathsRemap, fileDescriptor) if err != nil { return nil, err } - if !isIncluded && !isFileImported { + if newFileDescriptor == nil { return nil, nil // Filtered out. } - if len(sourcePathsRemap) == 0 { + if newFileDescriptor == fileDescriptor { return imageFile, nil // No changes required. } - newFileDescriptor, err := remapFileDescriptor(fileDescriptor, sourcePathsRemap) - if err != nil { - return nil, err + fmt.Println("-----") + //b, _ := (&protojson.MarshalOptions{ + // Indent: " ", + //}).Marshal(newFileDescriptor) + //fmt.Println(string(b)) + //fmt.Println("FILE", newFileDescriptor.GetName()) + for _, filePath := range newFileDescriptor.Dependency { + fmt.Println(" IMPORT", filePath) + } + + // Remap the source code info. + if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { + newLocations := make([]*descriptorpb.SourceCodeInfo_Location, 0, len(locations)) + for _, location := range locations { + oldPath := location.Path + newPath, noComment := sourcePathsRemap.newPath(oldPath) + if newPath == nil { + continue + } + if noComment || !slices.Equal(oldPath, newPath) { + location = shallowClone(location) + location.Path = newPath + } + if noComment { + location.LeadingDetachedComments = nil + location.LeadingComments = nil + location.TrailingComments = nil + } + newLocations = append(newLocations, location) + } + newFileDescriptor.SourceCodeInfo = &descriptorpb.SourceCodeInfo{ + Location: newLocations, + } } return bufimage.NewImageFile( newFileDescriptor, @@ -129,276 +170,369 @@ func filterImageFile( } type sourcePathsBuilder struct { - filePath string - imageIndex *imageIndex - filter *fullNameFilter - fileImports map[string]struct{} + imageIndex *imageIndex + closure *transitiveClosure + options *imageFilterOptions } -func addRemapsForFileDescriptor( +func (b *sourcePathsBuilder) remapFileDescriptor( sourcePathsRemap *sourcePathsRemapTrie, fileDescriptor *descriptorpb.FileDescriptorProto, - imageIndex *imageIndex, - filter *fullNameFilter, -) (bool, error) { +) (*descriptorpb.FileDescriptorProto, error) { fmt.Println("FILE", fileDescriptor.GetName()) packageName := protoreflect.FullName(fileDescriptor.GetPackage()) - if packageName != "" { - // Check if filtered by the package name. - if !filter.hasType(packageName) { - fmt.Println("FILTERED BY PACKAGE", packageName) - return false, nil - } + if !b.closure.hasType(fileDescriptor, b.options) { + fmt.Println(" FILTERED BY PACKAGE", packageName) + return nil, nil } - fileImports := make(map[string]struct{}) - builder := &sourcePathsBuilder{ - filePath: fileDescriptor.GetName(), - imageIndex: imageIndex, - filter: filter, - fileImports: fileImports, - } sourcePath := make(protoreflect.SourcePath, 0, 8) // Walk the file descriptor. - isIncluded := false - hasMessages, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, builder.addRemapsForDescriptor) + isDirty := false + fmt.Println("REMAP MESSAGES") + newMessages, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, b.remapDescriptor) + if err != nil { + return nil, err + } + fmt.Println("MESSAGES", len(newMessages)) + fmt.Println("MESSAGES", newMessages) + isDirty = isDirty || changed + newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, b.remapEnum) if err != nil { - return false, err + return nil, err } - isIncluded = isIncluded || hasMessages - hasEnums, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, builder.addRemapsForEnum) + isDirty = isDirty || changed + newServices, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileServicesTag), fileDescriptor.Service, b.remapService) if err != nil { - return false, err + return nil, err } - isIncluded = isIncluded || hasEnums - hasServices, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileServicesTag), fileDescriptor.Service, builder.addRemapsForService) + isDirty = isDirty || changed + // TODO: extension docs + //newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField) + newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField) if err != nil { - return false, err + return nil, err } - isIncluded = isIncluded || hasServices - hasExtensions, err := addRemapsForSlice(sourcePathsRemap, packageName, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, builder.addRemapsForField) + fmt.Println("CHANGED?", changed, "how many?", len(newExtensions)) + //if len(newExtensions) == 0 { + // sourcePathsRemap.markDeleted(append(sourcePath, fileExtensionsTag)) + //} + isDirty = isDirty || changed + newOptions, changed, err := remapMessage(nil, append(sourcePath, fileOptionsTag), fileDescriptor.Options, b.remapOptions) if err != nil { - return false, err + return nil, err } - isIncluded = isIncluded || hasExtensions - if err := builder.addRemapsForOptions(sourcePathsRemap, append(sourcePath, fileOptionsTag), fileDescriptor.Options); err != nil { - return false, err + isDirty = isDirty || changed + + if !isDirty { + fmt.Println(" NO CHANGES", fileDescriptor.GetName()) + return fileDescriptor, nil } + newFileDescriptor := shallowClone(fileDescriptor) + newFileDescriptor.MessageType = newMessages + newFileDescriptor.EnumType = newEnums + newFileDescriptor.Service = newServices + newFileDescriptor.Extension = newExtensions + newFileDescriptor.Options = newOptions + // Fix the imports to remove any that are no longer used. - // TODO: handle unused dependencies, and keep them? - if len(fileImports) != len(fileDescriptor.Dependency) { - indexTo := int32(0) - dependencyPath := append(sourcePath, fileDependencyTag) - dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) - for indexFrom, dependency := range fileDescriptor.Dependency { - path := append(dependencyPath, int32(indexFrom)) - if _, ok := fileImports[dependency]; ok { - dependencyChanges[indexFrom] = indexTo - if indexTo != int32(indexFrom) { - sourcePathsRemap.markMoved(path, indexTo) - } - indexTo++ - } else { - sourcePathsRemap.markDeleted(path) - dependencyChanges[indexFrom] = -1 - } - } - publicDependencyPath := append(sourcePath, filePublicDependencyTag) - for indexFrom, publicDependency := range fileDescriptor.PublicDependency { - path := append(publicDependencyPath, int32(indexFrom)) - indexTo := dependencyChanges[publicDependency] - if indexTo == -1 { - sourcePathsRemap.markDeleted(path) - } else if indexTo != int32(indexFrom) { + importsRequired := b.closure.imports[fileDescriptor.GetName()] + + indexTo := int32(0) + newFileDescriptor.Dependency = nil + dependencyPath := append(sourcePath, fileDependencyTag) + dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) + for indexFrom, importPath := range fileDescriptor.Dependency { + path := append(dependencyPath, int32(indexFrom)) + if importsRequired != nil && importsRequired.index(importPath) != -1 { + dependencyChanges[indexFrom] = indexTo + if indexTo != int32(indexFrom) { sourcePathsRemap.markMoved(path, indexTo) } + newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, importPath) + indexTo++ + // delete them as we go, so we know which ones weren't in the list + importsRequired.delete(importPath) + } else { + sourcePathsRemap.markDeleted(path) + dependencyChanges[indexFrom] = -1 } + } + if importsRequired != nil { + newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, importsRequired.keys()...) + } + + newFileDescriptor.PublicDependency = nil + publicDependencyPath := append(sourcePath, filePublicDependencyTag) + sourcePathsRemap.markDeleted(publicDependencyPath) + + // TODO: validate + newFileDescriptor.WeakDependency = nil + if len(fileDescriptor.WeakDependency) > 0 { weakDependencyPath := append(sourcePath, fileWeakDependencyTag) - for indexFrom, weakDependency := range fileDescriptor.WeakDependency { + for _, indexFrom := range fileDescriptor.WeakDependency { path := append(weakDependencyPath, int32(indexFrom)) - indexTo := dependencyChanges[weakDependency] + indexTo := dependencyChanges[indexFrom] if indexTo == -1 { sourcePathsRemap.markDeleted(path) - } else if indexTo != int32(indexFrom) { - sourcePathsRemap.markMoved(path, indexTo) + } else { + if indexTo != int32(indexFrom) { + sourcePathsRemap.markMoved(path, indexTo) + } + newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, indexTo) } } } - return isIncluded, nil + + return newFileDescriptor, nil } -func (b *sourcePathsBuilder) addRemapsForDescriptor( +func (b *sourcePathsBuilder) remapDescriptor( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, descriptor *descriptorpb.DescriptorProto, -) (bool, error) { - fullName := getFullName(parentName, descriptor) - mode := b.filter.inclusionMode(fullName) - if mode == inclusionModeNone { - // The type is excluded. - return false, nil - } - if mode == inclusionModeEnclosing { - fmt.Println("--- ENCLOSING:", fullName) +) (*descriptorpb.DescriptorProto, bool, error) { + if !b.closure.hasType(descriptor, b.options) { + fmt.Println(" FILTERED BY DESCRIPTOR", descriptor.GetName()) + return nil, true, nil + } + var newDescriptor *descriptorpb.DescriptorProto + isDirty := false + if mode := b.closure.elements[descriptor]; mode == inclusionModeEnclosing { + fmt.Println("--- ENCLOSING:", descriptor.GetName()) // If the type is only enclosing, only the namespace matters. + isDirty = true + newDescriptor = shallowClone(descriptor) + sourcePathsRemap.markNoComment(sourcePath) sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageReservedRangesTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageReservedNamesTag)) + newDescriptor.Field = nil + newDescriptor.OneofDecl = nil + newDescriptor.ExtensionRange = nil + newDescriptor.ReservedRange = nil + newDescriptor.ReservedName = nil } else { - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.addRemapsForField); err != nil { - return false, err + newFields, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.remapField) + if err != nil { + return nil, false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.addRemapsForOneof); err != nil { - return false, err + isDirty = isDirty || changed + newOneofs, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.remapOneof) + if err != nil { + return nil, false, err } - for index, extensionRange := range descriptor.GetExtensionRange() { - extensionRangeOptionsPath := append(sourcePath, messageExtensionRangesTag, int32(index), extensionRangeOptionsTag) - if err := b.addRemapsForOptions(sourcePathsRemap, extensionRangeOptionsPath, extensionRange.GetOptions()); err != nil { - return false, err - } + isDirty = isDirty || changed + newExtensionRange, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionRangesTag), descriptor.ExtensionRange, b.remapExtensionRange) + if err != nil { + return nil, false, err } + isDirty = isDirty || changed + if isDirty { + newDescriptor = shallowClone(descriptor) + newDescriptor.Field = newFields + newDescriptor.OneofDecl = newOneofs + newDescriptor.ExtensionRange = newExtensionRange + } + } + // TODO: sourcePath might not be correct here. + // TODO: extension docs. + newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.remapField) + if err != nil { + return nil, false, err + } + fmt.Println("DESCRIPTOR", changed, "how many?", len(newExtensions)) + //if len(newExtensions) == 0 { + // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionsTag)) + //} + isDirty = isDirty || changed + newDescriptors, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.remapDescriptor) + if err != nil { + return nil, false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.addRemapsForField); err != nil { - return false, err + isDirty = isDirty || changed + newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.remapEnum) + if err != nil { + return nil, false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.addRemapsForDescriptor); err != nil { - return false, err + isDirty = isDirty || changed + newOptions, changed, err := remapMessage(nil, append(sourcePath, messageOptionsTag), descriptor.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err } - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.addRemapsForEnum); err != nil { - return false, err + isDirty = isDirty || changed + + if !isDirty { + fmt.Println(" NO CHANGES", descriptor.GetName()) + return descriptor, false, nil + } + if newDescriptor == nil { + newDescriptor = shallowClone(descriptor) + } + newDescriptor.Extension = newExtensions + newDescriptor.NestedType = newDescriptors + newDescriptor.EnumType = newEnums + newDescriptor.Options = newOptions + return newDescriptor, true, nil +} + +func (b *sourcePathsBuilder) remapExtensionRange( + sourcePathsRemap *sourcePathsRemapTrie, + sourcePath protoreflect.SourcePath, + extensionRange *descriptorpb.DescriptorProto_ExtensionRange, +) (*descriptorpb.DescriptorProto_ExtensionRange, bool, error) { + newOptions, changed, err := remapMessage(nil, append(sourcePath, extensionRangeOptionsTag), extensionRange.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err } - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, messageOptionsTag), descriptor.GetOptions()); err != nil { - return false, err + if !changed { + return extensionRange, false, nil } - return true, nil + newExtensionRange := shallowClone(extensionRange) + newExtensionRange.Options = newOptions + return newExtensionRange, true, nil } -func (b *sourcePathsBuilder) addRemapsForEnum( +func (b *sourcePathsBuilder) remapEnum( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, enum *descriptorpb.EnumDescriptorProto, -) (bool, error) { - //fullName := b.imageIndex.ByDescriptor[enum] - fullName := getFullName(parentName, enum) - if !b.filter.hasType(fullName) { +) (*descriptorpb.EnumDescriptorProto, bool, error) { + if !b.closure.hasType(enum, b.options) { // The type is excluded, enum values cannot be excluded individually. - return false, nil + return nil, true, nil } - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, enumOptionsTag), enum.GetOptions()); err != nil { - return false, err + var isDirty bool + newOptions, changed, err := remapMessage(nil, append(sourcePath, enumOptionsTag), enum.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err } + isDirty = changed // Walk the enum values. - for index, enumValue := range enum.Value { - enumValuePath := append(sourcePath, enumValuesTag, int32(index)) - enumValueOptionsPath := append(enumValuePath, enumValueOptionsTag) - if err := b.addRemapsForOptions(sourcePathsRemap, enumValueOptionsPath, enumValue.GetOptions()); err != nil { - return false, err - } + newEnumValues, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, enumValuesTag), enum.Value, b.remapEnumValue) + if err != nil { + return nil, false, err + } + isDirty = isDirty || changed + if !isDirty { + return enum, true, nil + } + newEnum := shallowClone(enum) + newEnum.Options = newOptions + newEnum.Value = newEnumValues + return newEnum, true, nil +} + +func (b *sourcePathsBuilder) remapEnumValue( + sourcePathsRemap *sourcePathsRemapTrie, + sourcePath protoreflect.SourcePath, + enumValue *descriptorpb.EnumValueDescriptorProto, +) (*descriptorpb.EnumValueDescriptorProto, bool, error) { + newOptions, changed, err := remapMessage(nil, append(sourcePath, enumValueOptionsTag), enumValue.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err + } + if !changed { + return enumValue, false, nil } - return true, nil + newEnumValue := shallowClone(enumValue) + newEnumValue.Options = newOptions + return newEnumValue, true, nil } -func (b *sourcePathsBuilder) addRemapsForOneof( +func (b *sourcePathsBuilder) remapOneof( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, oneof *descriptorpb.OneofDescriptorProto, -) (bool, error) { - fullName := getFullName(parentName, oneof) - if !b.filter.hasType(fullName) { - // The type is excluded, oneof fields cannot be excluded individually. - return false, nil +) (*descriptorpb.OneofDescriptorProto, bool, error) { + options, changed, err := remapMessage(nil, append(sourcePath, oneofOptionsTag), oneof.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err } - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, oneofOptionsTag), oneof.GetOptions()); err != nil { - return false, err + if !changed { + return oneof, false, nil } - return true, nil + newOneof := shallowClone(oneof) + newOneof.Options = options + return newOneof, true, nil } -func (b *sourcePathsBuilder) addRemapsForService( +func (b *sourcePathsBuilder) remapService( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, service *descriptorpb.ServiceDescriptorProto, -) (bool, error) { - fullName := getFullName(parentName, service) - if !b.filter.hasType(fullName) { - // The type is excluded. - return false, nil +) (*descriptorpb.ServiceDescriptorProto, bool, error) { + if !b.closure.hasType(service, b.options) { + return nil, true, nil } + isDirty := false // Walk the service methods. - if _, err := addRemapsForSlice(sourcePathsRemap, fullName, append(sourcePath, serviceMethodsTag), service.Method, b.addRemapsForMethod); err != nil { - return false, err + newMethods, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, serviceMethodsTag), service.Method, b.remapMethod) + if err != nil { + return nil, false, err + } + isDirty = isDirty || changed + newOptions, changed, err := remapMessage(nil, append(sourcePath, serviceOptionsTag), service.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err } - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, serviceOptionsTag), service.GetOptions()); err != nil { - return false, err + isDirty = isDirty || changed + if !isDirty { + return service, false, nil } - return true, nil + newService := shallowClone(service) + newService.Method = newMethods + newService.Options = newOptions + return newService, true, nil } -func (b *sourcePathsBuilder) addRemapsForMethod( +func (b *sourcePathsBuilder) remapMethod( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, method *descriptorpb.MethodDescriptorProto, -) (bool, error) { - fullName := getFullName(parentName, method) - if !b.filter.hasType(fullName) { - // The type is excluded. - return false, nil - } - inputName := protoreflect.FullName(strings.TrimPrefix(method.GetInputType(), ".")) - if !b.filter.hasType(inputName) { - // The input type is excluded. - return false, fmt.Errorf("input type %s of method %s is excluded", inputName, fullName) - } - b.addRequiredType(inputName) - outputName := protoreflect.FullName(strings.TrimPrefix(method.GetOutputType(), ".")) - if !b.filter.hasType(outputName) { - // The output type is excluded. - return false, fmt.Errorf("output type %s of method %s is excluded", outputName, fullName) - } - b.addRequiredType(outputName) - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, methodOptionsTag), method.GetOptions()); err != nil { - return false, err - } - return true, nil +) (*descriptorpb.MethodDescriptorProto, bool, error) { + if !b.closure.hasType(method, b.options) { + return nil, true, nil + } + newOptions, changed, err := remapMessage(nil, append(sourcePath, methodOptionsTag), method.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err + } + if !changed { + return method, false, nil + } + newMethod := shallowClone(method) + newMethod.Options = newOptions + return newMethod, true, nil } -func (b *sourcePathsBuilder) addRemapsForField( +func (b *sourcePathsBuilder) remapField( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, field *descriptorpb.FieldDescriptorProto, -) (bool, error) { +) (*descriptorpb.FieldDescriptorProto, bool, error) { if field.Extendee != nil { - // This is an extension field. Extensions are filtered by the name. - extensionFullName := getFullName(parentName, field) - if !b.filter.hasType(extensionFullName) { - return false, nil + // Extensions are filtered by type. + fmt.Println("EXTEND", field.GetName()) + if !b.closure.hasType(field, b.options) { + return nil, true, nil } - extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) - b.addRequiredType(extendeeName) - - } else if b.filter.inclusionMode(parentName) == inclusionModeEnclosing { - return false, nil // The field is excluded. } switch field.GetType() { case descriptorpb.FieldDescriptorProto_TYPE_ENUM, descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, descriptorpb.FieldDescriptorProto_TYPE_GROUP: typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) - if !b.filter.hasType(typeName) { - return false, nil + typeInfo := b.imageIndex.ByName[typeName] + fmt.Println("FIELD", field.GetName(), "->", typeName) + if !b.closure.hasType(typeInfo.element, b.options) { + return nil, true, nil } - b.addRequiredType(typeName) case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, descriptorpb.FieldDescriptorProto_TYPE_FLOAT, descriptorpb.FieldDescriptorProto_TYPE_INT64, @@ -415,137 +549,107 @@ func (b *sourcePathsBuilder) addRemapsForField( descriptorpb.FieldDescriptorProto_TYPE_SINT32, descriptorpb.FieldDescriptorProto_TYPE_SINT64: default: - return false, fmt.Errorf("unknown field type %d", field.GetType()) + return nil, false, fmt.Errorf("unknown field type %d", field.GetType()) + } + newOption, changed, err := remapMessage(nil, append(sourcePath, fieldOptionsTag), field.GetOptions(), b.remapOptions) + if err != nil { + return nil, false, err } - if err := b.addRemapsForOptions(sourcePathsRemap, append(sourcePath, fieldOptionsTag), field.GetOptions()); err != nil { - return false, err + if !changed { + return field, false, nil } - return true, nil + newField := shallowClone(field) + newField.Options = newOption + return newField, true, nil } -func (b *sourcePathsBuilder) addRemapsForOptions( +func (b *sourcePathsBuilder) remapOptions( sourcePathsRemap *sourcePathsRemapTrie, optionsPath protoreflect.SourcePath, optionsMessage proto.Message, -) error { +) (proto.Message, bool, error) { if optionsMessage == nil { - return nil + return nil, false, nil } - var err error + var newOptions protoreflect.Message options := optionsMessage.ProtoReflect() numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - if !b.filter.hasOption(fd.FullName(), fd.IsExtension()) { + if !b.closure.hasOption(fd, b.options) { // Remove this option. optionPath := append(optionsPath, int32(fd.Number())) sourcePathsRemap.markDeleted(optionPath) + if newOptions == nil { + newOptions = shallowCloneReflect(options) + } + newOptions.Clear(fd) return true } numFieldsToKeep++ - if fd.IsExtension() { - // Add the extension type to the required types. - b.addRequiredType(fd.FullName()) - } - err = b.addOptionValue(fd, val) - return err == nil + return true }) if numFieldsToKeep == 0 { // No options to keep. sourcePathsRemap.markDeleted(optionsPath) + return nil, true, nil } - return err -} - -func (b *sourcePathsBuilder) addOptionValue( - fieldDescriptor protoreflect.FieldDescriptor, - value protoreflect.Value, -) error { - switch { - case fieldDescriptor.IsMap(): - if isMessageKind(fieldDescriptor.MapValue().Kind()) { - var err error - value.Map().Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool { - err = b.addOptionSingularValueForAny(v.Message()) - return err == nil - }) - return err - } - return nil - case isMessageKind(fieldDescriptor.Kind()): - if fieldDescriptor.IsList() { - listVal := value.List() - for i := 0; i < listVal.Len(); i++ { - if err := b.addOptionSingularValueForAny(listVal.Get(i).Message()); err != nil { - return err - } - } - return nil - } - return b.addOptionSingularValueForAny(value.Message()) - default: - return nil + if newOptions == nil { + return optionsMessage, false, nil } + return newOptions.Interface(), true, nil } -func (b *sourcePathsBuilder) addOptionSingularValueForAny(message protoreflect.Message) error { - md := message.Descriptor() - if md.FullName() == anyFullName { - // Found one! - typeURLFd := md.Fields().ByNumber(1) - if typeURLFd.Kind() != protoreflect.StringKind || typeURLFd.IsList() { - // should not be possible... - return nil - } - typeURL := message.Get(typeURLFd).String() - pos := strings.LastIndexByte(typeURL, '/') - msgType := protoreflect.FullName(typeURL[pos+1:]) - if !b.filter.hasType(msgType) { - return nil - } - b.addRequiredType(msgType) - // TODO: unmarshal the bytes to see if there are any nested Any messages - return nil - } - // keep digging - var err error - message.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - err = b.addOptionValue(fd, val) - return err == nil - }) - return err -} - -func (b *sourcePathsBuilder) addRequiredType(fullName protoreflect.FullName) { - info, ok := b.imageIndex.ByName[fullName] - if !ok { - panic(fmt.Sprintf("could not find file for %s", fullName)) +func remapMessage[T proto.Message]( + sourcePathsRemap *sourcePathsRemapTrie, + sourcePath protoreflect.SourcePath, + message T, + remapMessage func(*sourcePathsRemapTrie, protoreflect.SourcePath, proto.Message) (proto.Message, bool, error), +) (T, bool, error) { + var zeroValue T + newMessageOpaque, changed, err := remapMessage(sourcePathsRemap, sourcePath, message) + if err != nil { + return zeroValue, false, err } - file := info.imageFile - if file.Path() != b.filePath { - // This is an imported type. - b.fileImports[file.Path()] = struct{}{} + if newMessageOpaque == nil { + return zeroValue, true, nil } + if !changed { + return message, false, nil + } + newMessage, _ := newMessageOpaque.(T) // Safe to assert. + return newMessage, true, nil + } -func addRemapsForSlice[T any]( +func remapSlice[T any]( sourcePathsRemap *sourcePathsRemapTrie, - parentName protoreflect.FullName, sourcePath protoreflect.SourcePath, - list []T, - addRemapsForItem func(*sourcePathsRemapTrie, protoreflect.FullName, protoreflect.SourcePath, T) (bool, error), -) (bool, error) { + list []*T, + remapItem func(*sourcePathsRemapTrie, protoreflect.SourcePath, *T) (*T, bool, error), +) ([]*T, bool, error) { + isDirty := false + var newList []*T fromIndex, toIndex := int32(0), int32(0) for int(fromIndex) < len(list) { item := list[fromIndex] sourcePath := append(sourcePath, fromIndex) - isIncluded, err := addRemapsForItem(sourcePathsRemap, parentName, sourcePath, item) + newItem, changed, err := remapItem(sourcePathsRemap, sourcePath, item) if err != nil { - return false, err + return nil, false, err + } + isDirty = isDirty || changed + if isDirty && newList == nil { + newList = make([]*T, 0, len(list)) + newList = append(newList, list[:toIndex]...) } + isIncluded := newItem != nil if isIncluded { if fromIndex != toIndex { sourcePathsRemap.markMoved(sourcePath, toIndex) } + if isDirty { + newList = append(newList, newItem) + } toIndex++ } else { sourcePathsRemap.markDeleted(sourcePath) @@ -555,178 +659,10 @@ func addRemapsForSlice[T any]( if toIndex == 0 { sourcePathsRemap.markDeleted(sourcePath) } - return toIndex > 0, nil -} - -func remapFileDescriptor( - fileDescriptor *descriptorpb.FileDescriptorProto, - sourcePathRemaps sourcePathsRemapTrie, -) (*descriptorpb.FileDescriptorProto, error) { - fileDescriptorMessage := fileDescriptor.ProtoReflect() - newFileDescriptorMessage, err := remapMessageReflect(fileDescriptorMessage, sourcePathRemaps) - if err != nil { - return nil, err - } - newFileDescriptor, ok := newFileDescriptorMessage.Interface().(*descriptorpb.FileDescriptorProto) - if !ok { - return nil, syserror.Newf("unexpected type %T", newFileDescriptorMessage.Interface()) + if isDirty { + return newList, true, nil } - // Remap the source code info. - if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { - newLocations := make([]*descriptorpb.SourceCodeInfo_Location, 0, len(locations)) - for _, location := range locations { - oldPath := location.Path - newPath, noComment := sourcePathRemaps.newPath(oldPath) - if newPath == nil { - continue - } - if !slices.Equal(oldPath, newPath) || noComment { - location = shallowClone(location) - location.Path = newPath - } - if noComment { - location.LeadingDetachedComments = nil - location.LeadingComments = nil - location.TrailingComments = nil - } - newLocations = append(newLocations, location) - } - newFileDescriptor.SourceCodeInfo = &descriptorpb.SourceCodeInfo{ - Location: newLocations, - } - } - return newFileDescriptor, nil -} - -func remapMessageReflect( - message protoreflect.Message, - sourcePathRemaps sourcePathsRemapTrie, -) (protoreflect.Message, error) { - if len(sourcePathRemaps) == 0 { - return message, nil - } - if !message.IsValid() { - return nil, fmt.Errorf("invalid message %T", message) - } - fieldDescriptors, err := getFieldDescriptors(message, sourcePathRemaps) - if err != nil { - return nil, err - } - message = shallowCloneReflect(message) - for index, remapNode := range sourcePathRemaps { - fieldDescriptor := fieldDescriptors[index] - if fieldDescriptor == nil { - return nil, fmt.Errorf("missing field descriptor %d on type %s", remapNode.oldIndex, message.Descriptor().FullName()) - } - if remapNode.newIndex == -1 { - message.Clear(fieldDescriptor) - continue - } else if remapNode.newIndex != remapNode.oldIndex { - return nil, fmt.Errorf("unexpected field move %d to %d", remapNode.oldIndex, remapNode.newIndex) - } - value := message.Get(fieldDescriptor) - switch { - case fieldDescriptor.IsList(): - if len(remapNode.children) == 0 { - break - } - newList := message.NewField(fieldDescriptor).List() - if err := remapListReflect(newList, value.List(), remapNode.children); err != nil { - return nil, err - } - value = protoreflect.ValueOfList(newList) - case fieldDescriptor.IsMap(): - panic("map fields not yet supported") - default: - fieldMessage, err := remapMessageReflect(value.Message(), remapNode.children) - if err != nil { - return nil, err - } - value = protoreflect.ValueOfMessage(fieldMessage) - } - message.Set(fieldDescriptor, value) - } - return message, nil -} - -func remapListReflect( - dstList protoreflect.List, - srcList protoreflect.List, - sourcePathRemaps sourcePathsRemapTrie, -) error { - if len(sourcePathRemaps) == 0 { - return nil - } - toIndex := 0 - sourcePathIndex := 0 - for fromIndex := 0; fromIndex < srcList.Len(); fromIndex++ { - var remapNode *sourcePathsRemapTrieNode - for ; sourcePathIndex < len(sourcePathRemaps); sourcePathIndex++ { - nextRemapNode := sourcePathRemaps[sourcePathIndex] - if index := int(nextRemapNode.oldIndex); index > fromIndex { - break - } else if index == fromIndex { - remapNode = nextRemapNode - break - } - } - value := srcList.Get(fromIndex) - if remapNode == nil { - dstList.Append(value) - toIndex++ - continue - } - if remapNode.newIndex == -1 { - continue - } - if fromIndex != int(remapNode.oldIndex) || toIndex != int(remapNode.newIndex) { - return fmt.Errorf("unexpected list move %d to %d, expected %d to %d", remapNode.oldIndex, remapNode.newIndex, fromIndex, toIndex) - } - // If no children, the value is unchanged. - if len(remapNode.children) > 0 { - // Must be a list of messages to have children. - indexMessage, err := remapMessageReflect(value.Message(), remapNode.children) - if err != nil { - return err - } - value = protoreflect.ValueOfMessage(indexMessage) - } - dstList.Append(value) - toIndex++ - } - return nil -} - -func getFieldDescriptors( - message protoreflect.Message, - sourcePathRemaps sourcePathsRemapTrie, -) ([]protoreflect.FieldDescriptor, error) { - var hasExtension bool - fieldDescriptors := make([]protoreflect.FieldDescriptor, len(sourcePathRemaps)) - fields := message.Descriptor().Fields() - for index, remapNode := range sourcePathRemaps { - fieldDescriptor := fields.ByNumber(protoreflect.FieldNumber(remapNode.oldIndex)) - if fieldDescriptor == nil { - hasExtension = true - } else { - fieldDescriptors[index] = fieldDescriptor - } - } - if !hasExtension { - return fieldDescriptors, nil - } - message.Range(func(fieldDescriptor protoreflect.FieldDescriptor, _ protoreflect.Value) bool { - if !fieldDescriptor.IsExtension() { - return true // Skip non-extension fields. - } - if index, found := sort.Find(len(sourcePathRemaps), func(i int) int { - return int(fieldDescriptor.Number()) - int(sourcePathRemaps[i].oldIndex) - }); found { - fieldDescriptors[index] = fieldDescriptor - } - return true - }) - return fieldDescriptors, nil + return list, false, nil } func shallowClone[T proto.Message](src T) T { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter_test.go b/private/bufpkg/bufimage/bufimageutil/image_filter_test.go index 2d0c67c7f6..93f5b10a1d 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter_test.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter_test.go @@ -14,7 +14,7 @@ package bufimageutil -import ( +/*import ( "bytes" "context" "sort" @@ -273,4 +273,4 @@ func testFilterOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, ima generated := txtar.Format(archive) expectedFile := t.Name() + ".txtar" checkExpectation(t, ctx, generated, bucket, expectedFile) -} +}*/ diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index ea531f902a..99cea8a303 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -35,16 +35,17 @@ type imageIndex struct { // ByName maps fully qualified type names to information about the named // element. ByName map[protoreflect.FullName]elementInfo - // Files maps file names to the file descriptor protos. - Files map[string]*descriptorpb.FileDescriptorProto // NameToExtensions maps fully qualified type names to all known // extension definitions for a type name. - NameToExtensions map[string][]*descriptorpb.FieldDescriptorProto + NameToExtensions map[protoreflect.FullName][]*descriptorpb.FieldDescriptorProto // NameToOptions maps `google.protobuf.*Options` type names to their // known extensions by field tag. - NameToOptions map[string]map[int32]*descriptorpb.FieldDescriptorProto + NameToOptions map[protoreflect.FullName]map[int32]*descriptorpb.FieldDescriptorProto // Packages maps package names to package contents. Packages map[string]*packageInfo + // FileTypes maps file names to the fully qualified type names defined + // in the file. + FileTypes map[string][]protoreflect.FullName } type namedDescriptor interface { @@ -62,16 +63,15 @@ var _ namedDescriptor = (*descriptorpb.ServiceDescriptorProto)(nil) var _ namedDescriptor = (*descriptorpb.MethodDescriptorProto)(nil) type elementInfo struct { - fullName protoreflect.FullName - imageFile bufimage.ImageFile //string // TODO: maybe bufimage.ImageFile? - parentName protoreflect.FullName //namedDescriptor - element namedDescriptor + fullName protoreflect.FullName + file bufimage.ImageFile + parent namedDescriptor + element namedDescriptor } type packageInfo struct { fullName protoreflect.FullName files []bufimage.ImageFile - types []protoreflect.FullName subPackages []*packageInfo } @@ -80,14 +80,14 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* index := &imageIndex{ ByName: make(map[protoreflect.FullName]elementInfo), ByDescriptor: make(map[namedDescriptor]elementInfo), - Files: make(map[string]*descriptorpb.FileDescriptorProto), + FileTypes: make(map[string][]protoreflect.FullName), Packages: make(map[string]*packageInfo), } if options.includeCustomOptions { - index.NameToOptions = make(map[string]map[int32]*descriptorpb.FieldDescriptorProto) + index.NameToOptions = make(map[protoreflect.FullName]map[int32]*descriptorpb.FieldDescriptorProto) } if options.includeKnownExtensions { - index.NameToExtensions = make(map[string][]*descriptorpb.FieldDescriptorProto) + index.NameToExtensions = make(map[protoreflect.FullName][]*descriptorpb.FieldDescriptorProto) } for _, imageFile := range image.Files() { @@ -95,7 +95,12 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* pkg.files = append(pkg.files, imageFile) fileName := imageFile.Path() fileDescriptorProto := imageFile.FileDescriptorProto() - index.Files[fileName] = fileDescriptorProto + index.ByDescriptor[fileDescriptorProto] = elementInfo{ + fullName: pkg.fullName, + file: imageFile, + element: fileDescriptorProto, + parent: nil, + } err := walk.DescriptorProtos(fileDescriptorProto, func(name protoreflect.FullName, msg proto.Message) error { if _, existing := index.ByName[name]; existing { return fmt.Errorf("duplicate for %q", name) @@ -104,9 +109,13 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* if !ok { return fmt.Errorf("unexpected descriptor type %T", msg) } - var parentName protoreflect.FullName + var parent namedDescriptor = fileDescriptorProto if pos := strings.LastIndexByte(string(name), '.'); pos != -1 { - parentName = name[:pos] + parent = index.ByName[name[:pos]].element + if parent == nil { + // parent name was a package name, not an element name + parent = fileDescriptorProto + } } // certain descriptor types don't need to be indexed: @@ -120,7 +129,7 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* if d.Extendee == nil { return nil } - extendeeName := strings.TrimPrefix(d.GetExtendee(), ".") + extendeeName := protoreflect.FullName(strings.TrimPrefix(d.GetExtendee(), ".")) if options.includeCustomOptions && isOptionsTypeName(extendeeName) { if _, ok := index.NameToOptions[extendeeName]; !ok { index.NameToOptions[extendeeName] = make(map[int32]*descriptorpb.FieldDescriptorProto) @@ -133,14 +142,14 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* } info := elementInfo{ - fullName: name, - imageFile: imageFile, - parentName: parentName, - element: descriptor, + fullName: name, + file: imageFile, + parent: parent, + element: descriptor, } index.ByName[name] = info index.ByDescriptor[descriptor] = info - pkg.types = append(pkg.types, name) + index.FileTypes[fileName] = append(index.FileTypes[fileName], name) return nil }) @@ -172,7 +181,7 @@ func addPackageToIndex(pkgName string, index *imageIndex) *packageInfo { return pkg } -func isOptionsTypeName(typeName string) bool { +func isOptionsTypeName(typeName protoreflect.FullName) bool { switch typeName { case "google.protobuf.FileOptions", "google.protobuf.MessageOptions", diff --git a/private/bufpkg/bufimage/bufimageutil/image_types.go b/private/bufpkg/bufimage/bufimageutil/image_types.go deleted file mode 100644 index 648773a135..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/image_types.go +++ /dev/null @@ -1,566 +0,0 @@ -// Copyright 2020-2025 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimageutil - -import ( - "fmt" - "strings" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/types/descriptorpb" -) - -type inclusionMode int - -const ( - inclusionModeNone inclusionMode = iota - inclusionModeEnclosing - inclusionModeExplicit -) - -type fullNameFilter struct { - options *imageFilterOptions - index *imageIndex - includes map[protoreflect.FullName]inclusionMode - excludes map[protoreflect.FullName]struct{} - depth int -} - -func newFullNameFilter( - imageIndex *imageIndex, - options *imageFilterOptions, -) (*fullNameFilter, error) { - filter := &fullNameFilter{ - options: options, - index: imageIndex, - } - if !options.includeCustomOptions && len(options.includeOptions) > 0 { - return nil, fmt.Errorf("cannot include options without including custom options") - } - for excludeType := range options.excludeTypes { - excludeType := protoreflect.FullName(excludeType) - if err := filter.checkFilterType(excludeType); err != nil { - return nil, err - } - if err := filter.exclude(excludeType); err != nil { - return nil, err - } - } - for includeType := range options.includeTypes { - includeType := protoreflect.FullName(includeType) - if err := filter.checkFilterType(includeType); err != nil { - return nil, err - } - if err := filter.include(includeType); err != nil { - return nil, err - } - } - if err := filter.includeExtensions(); err != nil { - return nil, err - } - return filter, nil -} - -func (f *fullNameFilter) inclusionMode(fullName protoreflect.FullName) inclusionMode { - if f.excludes != nil { - if _, ok := f.excludes[fullName]; ok { - return inclusionModeNone - } - } - if f.includes != nil { - return f.includes[fullName] - } - return inclusionModeExplicit -} - -func (f *fullNameFilter) hasType(fullName protoreflect.FullName) (isIncluded bool) { - defer fmt.Println("hasType", fullName, isIncluded) - return f.inclusionMode(fullName) != inclusionModeNone -} - -func (f *fullNameFilter) hasOption(fullName protoreflect.FullName, isExtension bool) (isIncluded bool) { - defer fmt.Println("hasOption", fullName, isIncluded) - if f.options.excludeOptions != nil { - if _, ok := f.options.excludeOptions[string(fullName)]; ok { - return false - } - } - if !f.options.includeCustomOptions { - return !isExtension - } - if f.options.includeOptions != nil { - _, ok := f.options.includeOptions[string(fullName)] - return ok - } - return true -} - -func (f *fullNameFilter) isExplicitExclude(fullName protoreflect.FullName) bool { - if f.excludes == nil { - return false - } - _, ok := f.excludes[fullName] - return ok -} - -func (f *fullNameFilter) exclude(fullName protoreflect.FullName) error { - fmt.Println("exclude", fullName) - if _, excluded := f.excludes[fullName]; excluded { - return nil - } - if descriptorInfo, ok := f.index.ByName[fullName]; ok { - return f.excludeElement(fullName, descriptorInfo.element) - } - packageInfo, ok := f.index.Packages[string(fullName)] - if !ok { - return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) - } - for _, file := range packageInfo.files { - // Remove the package name from the excludes since it is not unique per file. - delete(f.excludes, fullName) - if err := f.excludeElement(fullName, file.FileDescriptorProto()); err != nil { - return err - } - } - for _, subPackage := range packageInfo.subPackages { - if err := f.exclude(subPackage.fullName); err != nil { - return err - } - } - return nil -} - -func (f *fullNameFilter) excludeElement(fullName protoreflect.FullName, descriptor namedDescriptor) error { - if _, excluded := f.excludes[fullName]; excluded { - return nil - } - if f.excludes == nil { - f.excludes = make(map[protoreflect.FullName]struct{}) - } - f.excludes[fullName] = struct{}{} - switch descriptor := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - if err := forEachDescriptor(fullName, descriptor.GetMessageType(), f.excludeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.excludeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetService(), f.excludeElement); err != nil { - return err - } - return nil - case *descriptorpb.DescriptorProto: - // Exclude all sub-elements - if err := forEachDescriptor(fullName, descriptor.GetNestedType(), f.excludeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.excludeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetOneofDecl(), f.excludeElement); err != nil { - return err - } - return nil - case *descriptorpb.EnumDescriptorProto: - // Value is excluded by parent. - return nil - case *descriptorpb.OneofDescriptorProto: - return nil - case *descriptorpb.ServiceDescriptorProto: - if err := forEachDescriptor(fullName, descriptor.GetMethod(), f.excludeElement); err != nil { - return err - } - return nil - case *descriptorpb.MethodDescriptorProto: - return nil - default: - return errorUnsupportedFilterType(descriptor, fullName) - } -} - -func (f *fullNameFilter) include(fullName protoreflect.FullName) error { - fmt.Println("include", fullName) - if _, included := f.includes[fullName]; included { - return nil - } - if descriptorInfo, ok := f.index.ByName[fullName]; ok { - if err := f.includeElement(fullName, descriptorInfo.element); err != nil { - return err - } - // Include the enclosing parent options. - fileDescriptor := descriptorInfo.imageFile.FileDescriptorProto() - if err := f.includeOptions(fileDescriptor); err != nil { - return err - } - // loop through all enclosing parents since nesting level - // could be arbitrarily deep - for parentName := descriptorInfo.parentName; parentName != ""; { - if isIncluded := f.hasType(parentName); isIncluded { - break - } - f.includes[parentName] = inclusionModeEnclosing - parentInfo, ok := f.index.ByName[parentName] - if !ok { - break - } - if err := f.includeOptions(parentInfo.element); err != nil { - return err - } - parentName = parentInfo.parentName - } - return nil - } - packageInfo, ok := f.index.Packages[string(fullName)] - if !ok { - return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) - } - for _, file := range packageInfo.files { - // Remove the package name from the includes since it is not unique per file. - delete(f.includes, fullName) - if err := f.includeElement(fullName, file.FileDescriptorProto()); err != nil { - return err - } - } - for _, subPackage := range packageInfo.subPackages { - if err := f.include(subPackage.fullName); err != nil { - return err - } - } - return nil -} - -func (f *fullNameFilter) includeElement(fullName protoreflect.FullName, descriptor namedDescriptor) error { - if _, included := f.includes[fullName]; included { - return nil - } - if f.isExplicitExclude(fullName) { - return nil // already excluded - } - if f.includes == nil { - f.includes = make(map[protoreflect.FullName]inclusionMode) - } - f.includes[fullName] = inclusionModeExplicit - - if err := f.includeOptions(descriptor); err != nil { - return err - } - - switch descriptor := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - if err := forEachDescriptor(fullName, descriptor.GetMessageType(), f.includeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.includeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetService(), f.includeElement); err != nil { - return err - } - return nil - case *descriptorpb.DescriptorProto: - if err := forEachDescriptor(fullName, descriptor.GetNestedType(), f.includeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetEnumType(), f.includeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetOneofDecl(), f.includeElement); err != nil { - return err - } - if err := forEachDescriptor(fullName, descriptor.GetField(), f.includeElement); err != nil { - return err - } - // Extensions are handled after all elements are included. - // This allows us to ensure that the extendee is included first. - return nil - case *descriptorpb.FieldDescriptorProto: - if descriptor.Extendee != nil { - // This is an extension field. - extendeeName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetExtendee(), ".")) - if err := f.include(extendeeName); err != nil { - return err - } - } - switch descriptor.GetType() { - case descriptorpb.FieldDescriptorProto_TYPE_ENUM, - descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, - descriptorpb.FieldDescriptorProto_TYPE_GROUP: - typeName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetTypeName(), ".")) - if err := f.include(typeName); err != nil { - return err - } - case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, - descriptorpb.FieldDescriptorProto_TYPE_FLOAT, - descriptorpb.FieldDescriptorProto_TYPE_INT64, - descriptorpb.FieldDescriptorProto_TYPE_UINT64, - descriptorpb.FieldDescriptorProto_TYPE_INT32, - descriptorpb.FieldDescriptorProto_TYPE_FIXED64, - descriptorpb.FieldDescriptorProto_TYPE_FIXED32, - descriptorpb.FieldDescriptorProto_TYPE_BOOL, - descriptorpb.FieldDescriptorProto_TYPE_STRING, - descriptorpb.FieldDescriptorProto_TYPE_BYTES, - descriptorpb.FieldDescriptorProto_TYPE_UINT32, - descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, - descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, - descriptorpb.FieldDescriptorProto_TYPE_SINT32, - descriptorpb.FieldDescriptorProto_TYPE_SINT64: - default: - return fmt.Errorf("unknown field type %d", descriptor.GetType()) - } - return nil - case *descriptorpb.EnumDescriptorProto: - for _, enumValue := range descriptor.GetValue() { - if err := f.includeOptions(enumValue); err != nil { - return err - } - } - return nil - case *descriptorpb.OneofDescriptorProto: - return nil - case *descriptorpb.ServiceDescriptorProto: - if err := forEachDescriptor(fullName, descriptor.GetMethod(), f.includeElement); err != nil { - return err - } - return nil - case *descriptorpb.MethodDescriptorProto: - inputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetInputType(), ".")) - inputInfo, ok := f.index.ByName[inputName] - if !ok { - return fmt.Errorf("missing %q", inputName) - } - if err := f.includeElement(inputName, inputInfo.element); err != nil { - return err - } - - outputName := protoreflect.FullName(strings.TrimPrefix(descriptor.GetOutputType(), ".")) - outputInfo, ok := f.index.ByName[outputName] - if !ok { - return fmt.Errorf("missing %q", outputName) - } - if err := f.includeElement(outputName, outputInfo.element); err != nil { - return err - } - return nil - default: - return errorUnsupportedFilterType(descriptor, fullName) - } -} - -func (f *fullNameFilter) includeExtensions() error { - if !f.options.includeKnownExtensions || len(f.options.includeTypes) == 0 { - return nil // nothing to do - } - for extendeeName, extensions := range f.index.NameToExtensions { - extendeeName := protoreflect.FullName(extendeeName) - if f.inclusionMode(extendeeName) != inclusionModeExplicit { - continue - } - for _, extension := range extensions { - info := f.index.ByDescriptor[extension] - if f.hasType(info.fullName) { - continue - } - typeName := protoreflect.FullName(strings.TrimPrefix(extension.GetTypeName(), ".")) - if typeName != "" && !f.hasType(typeName) { - continue - } - if err := f.includeElement(info.fullName, extension); err != nil { - return err - } - } - } - return nil -} - -func (f *fullNameFilter) includeOptions(descriptor proto.Message) (err error) { - fmt.Println("includeOptions", descriptor.ProtoReflect().Descriptor().FullName()) - if !f.options.includeCustomOptions { - return nil - } - var optionsMessage proto.Message - switch descriptor := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.DescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.FieldDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.OneofDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.EnumDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.EnumValueDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.ServiceDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.MethodDescriptorProto: - optionsMessage = descriptor.GetOptions() - case *descriptorpb.DescriptorProto_ExtensionRange: - optionsMessage = descriptor.GetOptions() - default: - return fmt.Errorf("unexpected type for exploring options %T", descriptor) - } - if optionsMessage == nil { - return nil - } - options := optionsMessage.ProtoReflect() - optionsName := options.Descriptor().FullName() - optionsByNumber := f.index.NameToOptions[string(optionsName)] - options.Range(func(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) bool { - if !f.hasOption(fieldDescriptor.FullName(), fieldDescriptor.IsExtension()) { - return true - } - if err = f.includeOptionValue(fieldDescriptor, value); err != nil { - return false - } - if !fieldDescriptor.IsExtension() { - return true - } - extensionField, ok := optionsByNumber[int32(fieldDescriptor.Number())] - if !ok { - err = fmt.Errorf("cannot find ext no %d on %s", fieldDescriptor.Number(), optionsName) - return false - } - info := f.index.ByDescriptor[extensionField] - err = f.includeElement(info.fullName, extensionField) - return err == nil - }) - return err -} - -func (f *fullNameFilter) includeOptionValue(fieldDescriptor protoreflect.FieldDescriptor, value protoreflect.Value) error { - // If the value contains an Any message, we should add the message type - // therein to the closure. - switch { - case fieldDescriptor.IsMap(): - if isMessageKind(fieldDescriptor.MapValue().Kind()) { - var err error - value.Map().Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool { - err = f.includeOptionSingularValueForAny(v.Message()) - return err == nil - }) - return err - } - return nil - case isMessageKind(fieldDescriptor.Kind()): - if fieldDescriptor.IsList() { - listVal := value.List() - for i := 0; i < listVal.Len(); i++ { - if err := f.includeOptionSingularValueForAny(listVal.Get(i).Message()); err != nil { - return err - } - } - return nil - } - return f.includeOptionSingularValueForAny(value.Message()) - default: - return nil - } -} - -func (f *fullNameFilter) includeOptionSingularValueForAny(message protoreflect.Message) error { - md := message.Descriptor() - if md.FullName() == anyFullName { - // Found one! - typeURLFd := md.Fields().ByNumber(1) - if typeURLFd.Kind() != protoreflect.StringKind || typeURLFd.IsList() { - // should not be possible... - return nil - } - typeURL := message.Get(typeURLFd).String() - pos := strings.LastIndexByte(typeURL, '/') - msgType := protoreflect.FullName(typeURL[pos+1:]) - d, _ := f.index.ByName[msgType].element.(*descriptorpb.DescriptorProto) - if d != nil { - if err := f.includeElement(msgType, d); err != nil { - return err - } - } - // TODO: unmarshal the bytes to see if there are any nested Any messages - return nil - } - // keep digging - var err error - message.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - err = f.includeOptionValue(fd, val) - return err == nil - }) - return err -} - -func (f *fullNameFilter) checkFilterType(fullName protoreflect.FullName) error { - var isImport bool - if info, ok := f.index.ByName[fullName]; ok { - isImport = info.imageFile.IsImport() - } else if pkg, ok := f.index.Packages[string(fullName)]; ok { - for _, file := range pkg.files { - isImport = isImport || file.IsImport() - } - } else { - return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeNotFound) - } - if !f.options.allowImportedTypes && isImport { - return fmt.Errorf("type %q: %w", fullName, ErrImageFilterTypeIsImport) - } - return nil -} - -func forEachDescriptor[T namedDescriptor]( - parentName protoreflect.FullName, - list []T, - fn func(protoreflect.FullName, namedDescriptor) error, -) error { - for _, element := range list { - if err := fn(getFullName(parentName, element), element); err != nil { - return err - } - } - return nil -} - -func isMessageKind(k protoreflect.Kind) bool { - return k == protoreflect.MessageKind || k == protoreflect.GroupKind -} - -func errorUnsupportedFilterType(descriptor namedDescriptor, fullName protoreflect.FullName) error { - var descriptorType string - switch d := descriptor.(type) { - case *descriptorpb.FileDescriptorProto: - descriptorType = "file" - case *descriptorpb.DescriptorProto: - descriptorType = "message" - case *descriptorpb.FieldDescriptorProto: - if d.Extendee != nil { - descriptorType = "extension field" - } else { - descriptorType = "non-extension field" - } - case *descriptorpb.OneofDescriptorProto: - descriptorType = "oneof" - case *descriptorpb.EnumDescriptorProto: - descriptorType = "enum" - case *descriptorpb.EnumValueDescriptorProto: - descriptorType = "enum value" - case *descriptorpb.ServiceDescriptorProto: - descriptorType = "service" - case *descriptorpb.MethodDescriptorProto: - descriptorType = "method" - default: - descriptorType = fmt.Sprintf("%T", d) - } - return fmt.Errorf("%s is unsupported filter type: %s", fullName, descriptorType) -} From 99902e6bd1129876c51370129c1d06d38418c76c Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 19:01:14 +0100 Subject: [PATCH 12/80] Cleanup testcase --- .../testdata/excludeoptions/a.proto | 82 --- .../testdata/excludeoptions/options.proto | 56 -- .../bufimageutil/testdata/filtertypes/a.proto | 31 - .../bufimageutil/testdata/filtertypes/b.proto | 0 .../TestExcludeOptionImports/ExcludeBar.txtar | 0 .../TestExcludeOptionImports/ExcludeFoo.txtar | 0 .../ExcludeFooBar.txtar | 0 .../NoneExcluded.txtar | 0 .../{excludeoptionimports => imports}/a.proto | 0 .../bufimageutil/testdata/imports/bar.txtar | 401 +++++++++++++ .../bufimageutil/testdata/imports/foo.txtar | 485 +++++++++++++++ .../testdata/imports/foo_bar.txtar | 9 + .../testdata/imports/none_excluded.txtar | 486 +++++++++++++++ .../options.proto | 0 .../ExcludeBar.txtar => enum.exclude.txtar} | 12 +- .../testdata/nesting/message.exclude.txtar | 17 + .../bufimageutil/testdata/nesting/mixed.txtar | 10 + .../nesting/recursenested.exclude.txtar | 28 + ...ludeBar.txtar => usingother.exclude.txtar} | 5 + .../all.exclude.txtar} | 31 +- .../options.foo.exclude.txtar} | 9 - .../options/options.foo.include.txtar | 463 +++++++++++++++ .../options.jstype.exclude.txtar} | 19 +- .../options/options.jstype.include.txtar | 454 ++++++++++++++ .../options/options.message.include.txtar | 456 +++++++++++++++ .../options.only_file.txtar} | 8 - .../pkg.Foo.exclude.txtar} | 42 +- .../options/pkg.FooEnum.exclude.txtar | 552 ++++++++++++++++++ .../pkg.FooService.Do.exclude.txtar} | 19 +- .../pkg.FooService.exclude.txtar} | 31 +- .../options/pkg.FooService.mixed.txtar | 213 +++++++ 31 files changed, 3625 insertions(+), 294 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/b.proto rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptionimports => imports}/TestExcludeOptionImports/ExcludeBar.txtar (100%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptionimports => imports}/TestExcludeOptionImports/ExcludeFoo.txtar (100%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptionimports => imports}/TestExcludeOptionImports/ExcludeFooBar.txtar (100%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptionimports => imports}/TestExcludeOptionImports/NoneExcluded.txtar (100%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptionimports => imports}/a.proto (100%) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/foo_bar.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptionimports => imports}/options.proto (100%) rename private/bufpkg/bufimage/bufimageutil/testdata/nesting/{TestFilterTypes/ExcludeBar.txtar => enum.exclude.txtar} (65%) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/nesting/message.exclude.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/nesting/mixed.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/nesting/recursenested.exclude.txtar rename private/bufpkg/bufimage/bufimageutil/testdata/nesting/{TestFilterTypes/IncludeBar.txtar => usingother.exclude.txtar} (77%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/ExcludeAll.txtar => options/all.exclude.txtar} (96%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/ExcludeFoo.txtar => options/options.foo.exclude.txtar} (98%) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/NoneExcluded.txtar => options/options.jstype.exclude.txtar} (97%) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/OnlyFile.txtar => options/options.only_file.txtar} (99%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/OnlyOneOf.txtar => options/pkg.Foo.exclude.txtar} (96%) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooEnum.exclude.txtar rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/ExcludeMessage.txtar => options/pkg.FooService.Do.exclude.txtar} (97%) rename private/bufpkg/bufimage/bufimageutil/testdata/{excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar => options/pkg.FooService.exclude.txtar} (96%) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.mixed.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto deleted file mode 100644 index 04f28c525b..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/a.proto +++ /dev/null @@ -1,82 +0,0 @@ -syntax = "proto2"; -package pkg; -import "options.proto"; - -option (UsedOption.file_foo).foo = "str"; -option (UsedOption.file_baz) = "str"; - -message Foo { - option (message_foo).foo = "str"; - option (message_baz) = "str"; - - optional uint64 foo = 1 [ - (field_foo).foo = "str", - (field_baz) = "str", - jstype = JS_STRING - ]; - - oneof testOneof { - option (oneof_foo).foo = "str"; - option (oneof_baz) = "str"; - - string bar = 2; - bytes baz = 3; - } - - enum FooFooEnum { - option (enum_foo).foo = "str"; - option (enum_baz) = "str"; - - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1 [ - (enum_value_foo).foo = "str", - (enum_value_baz) = "str" - ]; - } - - message Bar { - optional string bar = 1 [ - (field_foo).foo = "str", - (field_baz) = "str" - ]; - } - optional Bar nested_bar = 4; - - extensions 10 to max; -} - -enum FooEnum { - option (enum_foo).foo = "str"; - option (enum_baz) = "str"; - option deprecated = true; - - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1 [ - (enum_value_foo).foo = "str", - (enum_value_baz) = "str" - ]; -} - -message Empty{} - -service FooService { - option (service_foo).foo = "str"; - option (service_baz) = "str"; - - rpc Do(Empty) returns (Empty) { - option (method_foo).foo = "str"; - option (method_baz) = "str"; - }; - - rpc DoNot(Empty) returns (Empty) { - option (method_foo).foo = "str"; - option (method_baz) = "str"; - }; -} - -extend Foo { - optional string extension = 11 [ - (field_foo).foo = "str", - (field_baz) = "str" - ]; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto b/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto deleted file mode 100644 index 57676254cb..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/options.proto +++ /dev/null @@ -1,56 +0,0 @@ -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; - -message UnusedOption { - string foo = 1; -} - -message UsedOption { - extend google.protobuf.FileOptions { - optional UsedOption file_foo = 50000; - optional UnusedOption file_bar = 50001; - optional string file_baz = 50002; - } - - string foo = 1; -} - -extend google.protobuf.MessageOptions { - optional UsedOption message_foo = 50000; - optional UnusedOption message_bar = 50001; - optional string message_baz = 50002; -} -extend google.protobuf.FieldOptions { - optional UsedOption field_foo = 50000; - optional UnusedOption field_bar = 50001; - optional string field_baz = 50002; -} -extend google.protobuf.OneofOptions { - optional UsedOption oneof_foo = 50000; - optional UnusedOption oneof_bar = 50001; - optional string oneof_baz = 50002; -} -extend google.protobuf.EnumOptions { - optional UsedOption enum_foo = 50000; - optional UnusedOption enum_bar = 50001; - optional string enum_baz = 50002; -} -extend google.protobuf.EnumValueOptions { - optional UsedOption enum_value_foo = 50000; - optional UnusedOption enum_value_bar = 50001; - optional string enum_value_baz = 50002; -} -extend google.protobuf.ServiceOptions { - optional UsedOption service_foo = 50000; - optional UnusedOption service_bar = 50001; - optional string service_baz = 50002; -} -extend google.protobuf.MethodOptions { - optional UsedOption method_foo = 50000; - optional UnusedOption method_bar = 50001; - optional string method_baz = 50002; -} - -message Files { - google.protobuf.FileDescriptorSet files = 1; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto deleted file mode 100644 index 7cffe7ce41..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/a.proto +++ /dev/null @@ -1,31 +0,0 @@ -syntax = "proto3"; -package pkg; - -message A { - string a = 1; - message NestedA { - message NestedNestedA { - string nested_nested_x = 1; - } - string nested_x = 1; - } - NestedA nested_a = 1; -} - -enum FooEnum { - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1; -} - -message B { - enum NestedBEnum { - NESTED_B_ENUM_X = 0; - NESTED_B_ENUM_Y = 1; - } - FooEnum foo_enum = 1; - Foo.NestedA nested_a = 2; -} - -message Baz { - Bar.NestedBarEnum nested_bar_enum = 1; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/b.proto b/private/bufpkg/bufimage/bufimageutil/testdata/filtertypes/b.proto deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeBar.txtar similarity index 100% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeBar.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeBar.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFoo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFoo.txtar similarity index 100% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFoo.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFoo.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFooBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFooBar.txtar similarity index 100% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/ExcludeFooBar.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFooBar.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/NoneExcluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/NoneExcluded.txtar similarity index 100% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/TestExcludeOptionImports/NoneExcluded.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/NoneExcluded.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/imports/a.proto similarity index 100% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/a.proto rename to private/bufpkg/bufimage/bufimageutil/testdata/imports/a.proto diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar new file mode 100644 index 0000000000..bbf49af0f9 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar @@ -0,0 +1,401 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "options.proto"; +message Foo { + option (message_foo) = { foo: "str" }; + Bar nested_bar = 2; + message Bar { + string bar = 1; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2; + optional VerificationState verification = 3 [default = UNVERIFIED]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1; + optional EnumType enum_type = 2; + optional RepeatedFieldEncoding repeated_field_encoding = 3; + optional Utf8Validation utf8_validation = 4; + optional MessageEncoding message_encoding = 5; + optional JsonFormat json_format = 6; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994, 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1; + repeated int32 span = 2; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message OptionBar { + string bar = 1; +} +message OptionFoo { + string foo = 1; +} +extend google.protobuf.MessageOptions { + OptionFoo message_foo = 50000; + OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar new file mode 100644 index 0000000000..0706f98c00 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar @@ -0,0 +1,485 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "options.proto"; +message Foo { + Bar nested_bar = 2; + message Bar { + option (message_bar) = { bar: "str" }; + string bar = 1; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message OptionBar { + string bar = 1; +} +message OptionFoo { + string foo = 1; +} +extend google.protobuf.MessageOptions { + OptionFoo message_foo = 50000; + OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo_bar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo_bar.txtar new file mode 100644 index 0000000000..8209070bbc --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo_bar.txtar @@ -0,0 +1,9 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Foo { + Bar nested_bar = 2; + message Bar { + string bar = 1; + } +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar new file mode 100644 index 0000000000..1003545b2f --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar @@ -0,0 +1,486 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "options.proto"; +message Foo { + option (message_foo) = { foo: "str" }; + Bar nested_bar = 2; + message Bar { + option (message_bar) = { bar: "str" }; + string bar = 1; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message OptionBar { + string bar = 1; +} +message OptionFoo { + string foo = 1; +} +extend google.protobuf.MessageOptions { + OptionFoo message_foo = 50000; + OptionBar message_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/options.proto b/private/bufpkg/bufimage/bufimageutil/testdata/imports/options.proto similarity index 100% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptionimports/options.proto rename to private/bufpkg/bufimage/bufimageutil/testdata/imports/options.proto diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/enum.exclude.txtar similarity index 65% rename from private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/nesting/enum.exclude.txtar index d951fbac3f..e5108d3007 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/ExcludeBar.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/enum.exclude.txtar @@ -1,7 +1,15 @@ -- a.proto -- syntax = "proto3"; package pkg; +message Bar { + Foo.NestedFoo nested_foo = 2; + enum NestedBarEnum { + NESTED_BAR_ENUM_X = 0; + NESTED_BAR_ENUM_Y = 1; + } +} message Baz { + Bar.NestedBarEnum nested_bar_enum = 1; } message Foo { string x = 1; @@ -16,7 +24,3 @@ message Foo { } } } -enum FooEnum { - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/message.exclude.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/message.exclude.txtar new file mode 100644 index 0000000000..2a5c092a2d --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/message.exclude.txtar @@ -0,0 +1,17 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Bar { + FooEnum foo_enum = 1; + enum NestedBarEnum { + NESTED_BAR_ENUM_X = 0; + NESTED_BAR_ENUM_Y = 1; + } +} +message Baz { + Bar.NestedBarEnum nested_bar_enum = 1; +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/mixed.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/mixed.txtar new file mode 100644 index 0000000000..c580e345bd --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/mixed.txtar @@ -0,0 +1,10 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Foo { + string x = 1; +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/recursenested.exclude.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/recursenested.exclude.txtar new file mode 100644 index 0000000000..48a3ce2508 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/recursenested.exclude.txtar @@ -0,0 +1,28 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +message Bar { + FooEnum foo_enum = 1; + Foo.NestedFoo nested_foo = 2; + enum NestedBarEnum { + NESTED_BAR_ENUM_X = 0; + NESTED_BAR_ENUM_Y = 1; + } +} +message Baz { + Bar.NestedBarEnum nested_bar_enum = 1; +} +message Foo { + string x = 1; + NestedFoo nested_foo = 2; + message NestedButNotUsed { + string nested_but_not_used = 1; + } + message NestedFoo { + string nested_x = 1; + } +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/usingother.exclude.txtar similarity index 77% rename from private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/nesting/usingother.exclude.txtar index 72a9e3683f..bcff3d7385 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/nesting/TestFilterTypes/IncludeBar.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/nesting/usingother.exclude.txtar @@ -10,6 +10,11 @@ message Bar { } } message Foo { + string x = 1; + NestedFoo nested_foo = 2; + message NestedButNotUsed { + string nested_but_not_used = 1; + } message NestedFoo { string nested_x = 1; message NestedNestedFoo { diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/all.exclude.txtar similarity index 96% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/all.exclude.txtar index e71bbd516c..0e890cf7f5 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeAll.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/all.exclude.txtar @@ -1,36 +1,11 @@ -- a.proto -- syntax = "proto2"; package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; message Empty { } -message Foo { - optional uint64 foo = 1 [jstype = JS_STRING]; - oneof testOneof { - string bar = 2; - bytes baz = 3; - } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1; - } - enum FooFooEnum { - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1; - } - extensions 10 to max; -} -enum FooEnum { - option deprecated = true; - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1; -} -service FooService { - rpc Do ( Empty ) returns ( Empty ); - rpc DoNot ( Empty ) returns ( Empty ); -} -extend Foo { - optional string extension = 11; -} -- google/protobuf/descriptor.proto -- syntax = "proto2"; package google.protobuf; diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar similarity index 98% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar index 9c7c035296..384eaf7a15 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeFoo.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar @@ -13,15 +13,6 @@ message Foo { string bar = 2; bytes baz = 3; } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1 [(field_baz) = "str"]; - } - enum FooFooEnum { - option (enum_baz) = "str"; - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1 [(enum_value_baz) = "str"]; - } extensions 10 to max; } enum FooEnum { diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar new file mode 100644 index 0000000000..460b41c801 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar @@ -0,0 +1,463 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_foo) = { foo: "str" }; +message Empty { +} +message Foo { + option (message_foo) = { foo: "str" }; + optional uint64 foo = 1 [(field_foo) = { foo: "str" }]; + oneof testOneof { + option (oneof_foo) = { foo: "str" }; + string bar = 2; + bytes baz = 3; + } + extensions 10 to max; +} +enum FooEnum { + option (enum_foo) = { foo: "str" }; + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1 [(enum_value_foo) = { foo: "str" }]; +} +service FooService { + option (service_foo) = { foo: "str" }; + rpc Do ( Empty ) returns ( Empty ) { + option (method_foo) = { foo: "str" }; + } + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_foo) = { foo: "str" }; + } +} +extend Foo { + optional string extension = 11 [(field_foo) = { foo: "str" }]; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2; + optional VerificationState verification = 3 [default = UNVERIFIED]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1; + optional EnumType enum_type = 2; + optional RepeatedFieldEncoding repeated_field_encoding = 3; + optional Utf8Validation utf8_validation = 4; + optional MessageEncoding message_encoding = 5; + optional JsonFormat json_format = 6; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994, 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1; + repeated int32 span = 2; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar similarity index 97% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar index bcd0962533..d1f0974c84 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/NoneExcluded.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar @@ -9,30 +9,13 @@ message Empty { message Foo { option (message_baz) = "str"; option (message_foo) = { foo: "str" }; - optional uint64 foo = 1 [ - jstype = JS_STRING, - (field_baz) = "str", - (field_foo) = { foo: "str" } - ]; + optional uint64 foo = 1 [(field_baz) = "str", (field_foo) = { foo: "str" }]; oneof testOneof { option (oneof_baz) = "str"; option (oneof_foo) = { foo: "str" }; string bar = 2; bytes baz = 3; } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1 [(field_baz) = "str", (field_foo) = { foo: "str" }]; - } - enum FooFooEnum { - option (enum_baz) = "str"; - option (enum_foo) = { foo: "str" }; - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1 [ - (enum_value_baz) = "str", - (enum_value_foo) = { foo: "str" } - ]; - } extensions 10 to max; } enum FooEnum { diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar new file mode 100644 index 0000000000..2b83275219 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar @@ -0,0 +1,454 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +message Empty { +} +message Foo { + optional uint64 foo = 1 [jstype = JS_STRING]; + oneof testOneof { + string bar = 2; + bytes baz = 3; + } + extensions 10 to max; +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} +service FooService { + rpc Do ( Empty ) returns ( Empty ); + rpc DoNot ( Empty ) returns ( Empty ); +} +extend Foo { + optional string extension = 11; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2; + optional VerificationState verification = 3 [default = UNVERIFIED]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1; + optional EnumType enum_type = 2; + optional RepeatedFieldEncoding repeated_field_encoding = 3; + optional Utf8Validation utf8_validation = 4; + optional MessageEncoding message_encoding = 5; + optional JsonFormat json_format = 6; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994, 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1; + repeated int32 span = 2; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar new file mode 100644 index 0000000000..28352e873c --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar @@ -0,0 +1,456 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +message Empty { +} +message Foo { + option (message_baz) = "str"; + option (message_foo) = { foo: "str" }; + optional uint64 foo = 1; + oneof testOneof { + string bar = 2; + bytes baz = 3; + } + extensions 10 to max; +} +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} +service FooService { + rpc Do ( Empty ) returns ( Empty ); + rpc DoNot ( Empty ) returns ( Empty ); +} +extend Foo { + optional string extension = 11; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2; + optional VerificationState verification = 3 [default = UNVERIFIED]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1; + optional EnumType enum_type = 2; + optional RepeatedFieldEncoding repeated_field_encoding = 3; + optional Utf8Validation utf8_validation = 4; + optional MessageEncoding message_encoding = 5; + optional JsonFormat json_format = 6; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994, 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1; + repeated int32 span = 2; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar similarity index 99% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar index 8949934292..1f159f9e12 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyFile.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar @@ -12,14 +12,6 @@ message Foo { string bar = 2; bytes baz = 3; } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1; - } - enum FooFooEnum { - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1; - } extensions 10 to max; } enum FooEnum { diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.Foo.exclude.txtar similarity index 96% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.Foo.exclude.txtar index 3e3df1624a..65a44f4ff0 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyOneOf.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.Foo.exclude.txtar @@ -2,37 +2,31 @@ syntax = "proto2"; package pkg; import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; message Empty { } -message Foo { - optional uint64 foo = 1 [jstype = JS_STRING]; - oneof testOneof { - option (oneof_baz) = "str"; - option (oneof_foo) = { foo: "str" }; - string bar = 2; - bytes baz = 3; - } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1; - } - enum FooFooEnum { - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1; - } - extensions 10 to max; -} enum FooEnum { option deprecated = true; + option (enum_baz) = "str"; + option (enum_foo) = { foo: "str" }; FOO_ENUM_X = 0; - FOO_ENUM_Y = 1; + FOO_ENUM_Y = 1 [ + (enum_value_baz) = "str", + (enum_value_foo) = { foo: "str" } + ]; } service FooService { - rpc Do ( Empty ) returns ( Empty ); - rpc DoNot ( Empty ) returns ( Empty ); -} -extend Foo { - optional string extension = 11; + option (service_baz) = "str"; + option (service_foo) = { foo: "str" }; + rpc Do ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } } -- google/protobuf/descriptor.proto -- syntax = "proto2"; diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooEnum.exclude.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooEnum.exclude.txtar new file mode 100644 index 0000000000..89385adeed --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooEnum.exclude.txtar @@ -0,0 +1,552 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; +message Empty { +} +message Foo { + option (message_baz) = "str"; + option (message_foo) = { foo: "str" }; + optional uint64 foo = 1 [ + jstype = JS_STRING, + (field_baz) = "str", + (field_foo) = { foo: "str" } + ]; + oneof testOneof { + option (oneof_baz) = "str"; + option (oneof_foo) = { foo: "str" }; + string bar = 2; + bytes baz = 3; + } + extensions 10 to max; +} +service FooService { + option (service_baz) = "str"; + option (service_foo) = { foo: "str" }; + rpc Do ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } +} +extend Foo { + optional string extension = 11 [(field_baz) = "str", (field_foo) = { foo: "str" }]; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + repeated ExtensionRange extension_range = 5; + repeated FieldDescriptorProto extension = 6; + optional MessageOptions options = 7; + repeated OneofDescriptorProto oneof_decl = 8; + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } +} +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ExtensionRangeOptions { + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + message Declaration { + optional int32 number = 1; + optional string full_name = 2; + optional string type = 3; + optional bool reserved = 5; + optional bool repeated = 6; + reserved 4; + } + enum VerificationState { + DECLARATION = 0; + UNVERIFIED = 1; + } + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FeatureSetDefaults { + repeated FeatureSetEditionDefault defaults = 1; + optional Edition minimum_edition = 4; + optional Edition maximum_edition = 5; + message FeatureSetEditionDefault { + optional Edition edition = 3; + optional FeatureSet overridable_features = 4; + optional FeatureSet fixed_features = 5; + reserved 1, 2; + reserved "features"; + } +} +message FieldDescriptorProto { + optional string name = 1; + optional string extendee = 2; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string default_value = 7; + optional FieldOptions options = 8; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional bool proto3_optional = 17; + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + optional string syntax = 12; + optional Edition edition = 14; +} +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", + type: ".buf.descriptor.v1.FileDescriptorSetExtension" + } + ]; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + optional Semantic semantic = 5; + enum Semantic { + NONE = 0; + SET = 1; + ALIAS = 2; + } + } +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } + extensions 536000000 [ + declaration = { + number: 536000000, + full_name: ".buf.descriptor.v1.buf_source_code_info_extension", + type: ".buf.descriptor.v1.SourceCodeInfoExtension" + } + ]; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message Files { + google.protobuf.FileDescriptorSet files = 1; +} +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.Do.exclude.txtar similarity index 97% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.Do.exclude.txtar index 9c33d16124..31ef6f2cd1 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/ExcludeMessage.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.Do.exclude.txtar @@ -7,6 +7,8 @@ option (UsedOption.file_foo) = { foo: "str" }; message Empty { } message Foo { + option (message_baz) = "str"; + option (message_foo) = { foo: "str" }; optional uint64 foo = 1 [ jstype = JS_STRING, (field_baz) = "str", @@ -18,19 +20,6 @@ message Foo { string bar = 2; bytes baz = 3; } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1 [(field_baz) = "str", (field_foo) = { foo: "str" }]; - } - enum FooFooEnum { - option (enum_baz) = "str"; - option (enum_foo) = { foo: "str" }; - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1 [ - (enum_value_baz) = "str", - (enum_value_foo) = { foo: "str" } - ]; - } extensions 10 to max; } enum FooEnum { @@ -46,10 +35,6 @@ enum FooEnum { service FooService { option (service_baz) = "str"; option (service_foo) = { foo: "str" }; - rpc Do ( Empty ) returns ( Empty ) { - option (method_baz) = "str"; - option (method_foo) = { foo: "str" }; - } rpc DoNot ( Empty ) returns ( Empty ) { option (method_baz) = "str"; option (method_foo) = { foo: "str" }; diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude.txtar similarity index 96% rename from private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar rename to private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude.txtar index f89d852f21..6d617bde4b 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/excludeoptions/TestExcludeOptions/OnlyEnumValue.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude.txtar @@ -2,41 +2,38 @@ syntax = "proto2"; package pkg; import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; message Empty { } message Foo { - optional uint64 foo = 1 [jstype = JS_STRING]; + option (message_baz) = "str"; + option (message_foo) = { foo: "str" }; + optional uint64 foo = 1 [ + jstype = JS_STRING, + (field_baz) = "str", + (field_foo) = { foo: "str" } + ]; oneof testOneof { + option (oneof_baz) = "str"; + option (oneof_foo) = { foo: "str" }; string bar = 2; bytes baz = 3; } - optional Bar nested_bar = 4; - message Bar { - optional string bar = 1; - } - enum FooFooEnum { - FOOFOO_ENUM_X = 0; - FOOFOO_ENUM_Y = 1 [ - (enum_value_baz) = "str", - (enum_value_foo) = { foo: "str" } - ]; - } extensions 10 to max; } enum FooEnum { option deprecated = true; + option (enum_baz) = "str"; + option (enum_foo) = { foo: "str" }; FOO_ENUM_X = 0; FOO_ENUM_Y = 1 [ (enum_value_baz) = "str", (enum_value_foo) = { foo: "str" } ]; } -service FooService { - rpc Do ( Empty ) returns ( Empty ); - rpc DoNot ( Empty ) returns ( Empty ); -} extend Foo { - optional string extension = 11; + optional string extension = 11 [(field_baz) = "str", (field_foo) = { foo: "str" }]; } -- google/protobuf/descriptor.proto -- syntax = "proto2"; diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.mixed.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.mixed.txtar new file mode 100644 index 0000000000..8c996c168c --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.mixed.txtar @@ -0,0 +1,213 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; +message Empty { +} +service FooService { + option (service_baz) = "str"; + option (service_foo) = { foo: "str" }; + rpc DoNot ( Empty ) returns ( Empty ) { + option (method_baz) = "str"; + option (method_foo) = { foo: "str" }; + } +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + string file_baz = 50002; + } +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + string method_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + string service_baz = 50002; +} From b5b7daadab0f418f3af7fcb805a16c0288cf809b Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 19:02:10 +0100 Subject: [PATCH 13/80] Cleanup --- .../bufimage/bufimageutil/bufimageutil.go | 194 +++++++++--- .../bufimageutil/bufimageutil_test.go | 190 +++++++++--- .../bufimage/bufimageutil/image_filter.go | 60 ++-- .../bufimageutil/image_filter_test.go | 276 ------------------ .../bufimage/bufimageutil/image_index.go | 2 +- 5 files changed, 324 insertions(+), 398 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimageutil/image_filter_test.go diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 0fa8e24412..6e5ab90774 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -298,7 +298,6 @@ func (t *transitiveClosure) hasType( descriptor namedDescriptor, options *imageFilterOptions, ) (isIncluded bool) { - defer func() { fmt.Println("\thasType", descriptor.GetName(), isIncluded) }() if options == nil { return true // no filter } @@ -319,14 +318,12 @@ func (t *transitiveClosure) hasOption( fieldDescriptor protoreflect.FieldDescriptor, options *imageFilterOptions, ) (isIncluded bool) { - defer func() { fmt.Println("\thasOption", fieldDescriptor.FullName(), isIncluded) }() fullName := fieldDescriptor.FullName() - defer fmt.Println("\thasOption", fullName, isIncluded) if options == nil { return true // no filter } - if options.excludeTypes != nil { - if _, ok := options.excludeTypes[string(fullName)]; ok { + if options.excludeOptions != nil { + if _, ok := options.excludeOptions[string(fullName)]; ok { return false } } @@ -340,17 +337,7 @@ func (t *transitiveClosure) hasOption( return true } -func (t *transitiveClosure) excludeElement( - descriptor namedDescriptor, - imageIndex *imageIndex, - options *imageFilterOptions, -) error { - _ = descriptor - // TODO: implement - return nil -} - -func (t *transitiveClosure) addType( +func (t *transitiveClosure) includeType( typeName protoreflect.FullName, imageIndex *imageIndex, options *imageFilterOptions, @@ -384,7 +371,13 @@ func (t *transitiveClosure) addType( return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) } } - return t.addPackage(pkg, imageIndex, options) + for _, file := range pkg.files { + fileDescriptor := file.FileDescriptorProto() + if err := t.addElement(fileDescriptor, "", false, imageIndex, options); err != nil { + return err + } + } + return nil } func (t *transitiveClosure) addImport(fromPath, toPath string) { @@ -399,22 +392,6 @@ func (t *transitiveClosure) addImport(fromPath, toPath string) { imps.add(toPath) } -func (t *transitiveClosure) addPackage( - pkg *packageInfo, - imageIndex *imageIndex, - opts *imageFilterOptions, -) error { - fmt.Printf("ADD PACKAGE: %q\n", pkg.fullName) - for _, file := range pkg.files { - fmt.Println("\taddPackage", file.Path()) - fileDescriptor := file.FileDescriptorProto() - if err := t.addElement(fileDescriptor, "", false, imageIndex, opts); err != nil { - return err - } - } - return nil -} - func (t *transitiveClosure) addElement( descriptor namedDescriptor, referrerFile string, @@ -441,8 +418,6 @@ func (t *transitiveClosure) addElement( } // if this type is enclosed inside another, add enclosing types - fmt.Println("--- ADDING ELEMENT", descriptorInfo.fullName, "=>", t.elements[descriptor]) - fmt.Printf("\t %s %T\n", descriptorInfo.file.Path(), descriptorInfo.parent) if err := t.addEnclosing(descriptorInfo.parent, descriptorInfo.file.Path(), imageIndex, opts); err != nil { return err } @@ -469,9 +444,13 @@ func (t *transitiveClosure) addElement( case *descriptorpb.DescriptorProto: // Options and types for all fields for _, field := range typedDescriptor.GetField() { - if err := t.addFieldType(field, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + isIncluded, err := t.addFieldType(field, descriptorInfo.file.Path(), imageIndex, opts) + if err != nil { return err } + if !isIncluded { + continue + } if err := t.exploreCustomOptions(field, referrerFile, imageIndex, opts); err != nil { return err } @@ -523,7 +502,6 @@ func (t *transitiveClosure) addElement( } case *descriptorpb.FieldDescriptorProto: - fmt.Println("ADDING EXTENSION", typedDescriptor.GetExtendee()) // Regular fields are handled above in message descriptor case. // We should only find our way here for extensions. if typedDescriptor.Extendee == nil { @@ -537,12 +515,22 @@ func (t *transitiveClosure) addElement( if !ok { return fmt.Errorf("missing %q", extendeeName) } + if mode := t.elements[extendeeInfo.element]; mode == inclusionModeExcluded { + // The extendee is excluded, so this extension is also excluded. + t.elements[descriptor] = inclusionModeExcluded + return nil + } if err := t.addElement(extendeeInfo.element, descriptorInfo.file.Path(), impliedByCustomOption, imageIndex, opts); err != nil { return err } - if err := t.addFieldType(typedDescriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + isIncluded, err := t.addFieldType(typedDescriptor, descriptorInfo.file.Path(), imageIndex, opts) + if err != nil { return err } + if !isIncluded { + t.elements[descriptor] = inclusionModeExcluded + return nil + } default: return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) @@ -551,6 +539,116 @@ func (t *transitiveClosure) addElement( return nil } +func (t *transitiveClosure) excludeType( + typeName protoreflect.FullName, + imageIndex *imageIndex, + options *imageFilterOptions, +) error { + descriptorInfo, ok := imageIndex.ByName[typeName] + if ok { + // It's a type name + if !options.allowImportedTypes && descriptorInfo.file.IsImport() { + return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + } + return t.excludeElement(descriptorInfo.element, imageIndex, options) + } + // It could be a package name + pkg, ok := imageIndex.Packages[string(typeName)] + if !ok { + // but it's not... + return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeNotFound) + } + if !options.allowImportedTypes { + // if package includes only imported files, then reject + onlyImported := true + for _, file := range pkg.files { + if !file.IsImport() { + onlyImported = false + break + } + } + if onlyImported { + return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + } + } + // Exclude the package and all of its files. + for _, file := range pkg.files { + fileDescriptor := file.FileDescriptorProto() + if err := t.excludeElement(fileDescriptor, imageIndex, options); err != nil { + return err + } + } + return nil +} + +func (t *transitiveClosure) excludeElement( + descriptor namedDescriptor, + imageIndex *imageIndex, + opts *imageFilterOptions, +) error { + descriptorInfo := imageIndex.ByDescriptor[descriptor] + if existingMode, ok := t.elements[descriptor]; ok { + if existingMode != inclusionModeExcluded { + return fmt.Errorf("type %q is already included", descriptorInfo.fullName) + } + return nil + } + t.elements[descriptor] = inclusionModeExcluded + + switch descriptor := descriptor.(type) { + case *descriptorpb.FileDescriptorProto: + for _, descriptor := range descriptor.GetMessageType() { + if err := t.excludeElement(descriptor, imageIndex, opts); err != nil { + return err + } + } + for _, descriptor := range descriptor.GetEnumType() { + if err := t.excludeElement(descriptor, imageIndex, opts); err != nil { + return err + } + } + for _, descriptor := range descriptor.GetService() { + if err := t.excludeElement(descriptor, imageIndex, opts); err != nil { + return err + } + } + for _, extension := range descriptor.GetExtension() { + if err := t.excludeElement(extension, imageIndex, opts); err != nil { + return err + } + } + case *descriptorpb.DescriptorProto: + // Exclude all sub-elements + for _, descriptor := range descriptor.GetNestedType() { + if err := t.excludeElement(descriptor, imageIndex, opts); err != nil { + return err + } + } + for _, enumDescriptor := range descriptor.GetEnumType() { + if err := t.excludeElement(enumDescriptor, imageIndex, opts); err != nil { + return err + } + } + for _, extensionDescriptor := range descriptor.GetExtension() { + if err := t.excludeElement(extensionDescriptor, imageIndex, opts); err != nil { + return err + } + } + case *descriptorpb.EnumDescriptorProto: + // Enum values are not included in the closure, so nothing to do here. + case *descriptorpb.ServiceDescriptorProto: + for _, descriptor := range descriptor.GetMethod() { + if err := t.excludeElement(descriptor, imageIndex, opts); err != nil { + return err + } + } + case *descriptorpb.MethodDescriptorProto: + default: + return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) + } + return nil +} + func errorUnsupportedFilterType(descriptor namedDescriptor, fullName protoreflect.FullName) error { var descriptorType string switch d := descriptor.(type) { @@ -603,7 +701,7 @@ func (t *transitiveClosure) addEnclosing(descriptor namedDescriptor, enclosingFi return nil } -func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProto, referrerFile string, imageIndex *imageIndex, opts *imageFilterOptions) error { +func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProto, referrerFile string, imageIndex *imageIndex, opts *imageFilterOptions) (bool, error) { switch field.GetType() { case descriptorpb.FieldDescriptorProto_TYPE_ENUM, descriptorpb.FieldDescriptorProto_TYPE_MESSAGE, @@ -611,11 +709,15 @@ func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProt typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) info, ok := imageIndex.ByName[typeName] if !ok { - return fmt.Errorf("missing %q", typeName) + return false, fmt.Errorf("missing %q", typeName) + } + if mode := t.elements[info.element]; mode == inclusionModeExcluded { + // The field's type is excluded, so this field is also excluded. + return false, nil } err := t.addElement(info.element, referrerFile, false, imageIndex, opts) if err != nil { - return err + return false, err } case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE, descriptorpb.FieldDescriptorProto_TYPE_FLOAT, @@ -634,9 +736,9 @@ func (t *transitiveClosure) addFieldType(field *descriptorpb.FieldDescriptorProt descriptorpb.FieldDescriptorProto_TYPE_SINT64: // nothing to follow, custom options handled below. default: - return fmt.Errorf("unknown field type %d", field.GetType()) + return false, fmt.Errorf("unknown field type %d", field.GetType()) } - return nil + return true, nil } func (t *transitiveClosure) addExtensions( @@ -651,16 +753,13 @@ func (t *transitiveClosure) addExtensions( // we only collect extensions for messages that are directly reachable/referenced. continue } - fmt.Println("ADDING EXTENSIONS FOR", e.GetName()) msgDescriptor, ok := e.(*descriptorpb.DescriptorProto) if !ok { // not a message, nothing to do continue } - fmt.Println("\t", msgDescriptor.GetName()) descriptorInfo := imageIndex.ByDescriptor[msgDescriptor] for _, extendsDescriptor := range imageIndex.NameToExtensions[descriptorInfo.fullName] { - fmt.Println("\t\t", extendsDescriptor.GetName()) if err := t.addElement(extendsDescriptor, "", false, imageIndex, opts); err != nil { return err } @@ -706,6 +805,9 @@ func (t *transitiveClosure) exploreCustomOptions( optionsName := options.Descriptor().FullName() var err error options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { + //if !t.hasOption(fd, opts) { + // return true + //} // If the value contains an Any message, we should add the message type // therein to the closure. if err = t.exploreOptionValueForAny(fd, val, referrerFile, imageIndex, opts); err != nil { diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index f347648b99..a96841dce0 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -48,39 +48,65 @@ import ( var shouldUpdateExpectations = os.Getenv("BUFBUILD_BUF_BUFIMAGEUTIL_SHOULD_UPDATE_EXPECTATIONS") -func TestOptions(t *testing.T) { +func TestTypes(t *testing.T) { t.Parallel() t.Run("message", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.Foo"}, "pkg.Foo.txtar") + runDiffTest(t, "testdata/options", "pkg.Foo.txtar", WithIncludeTypes("pkg.Foo")) }) t.Run("enum", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.FooEnum"}, "pkg.FooEnum.txtar") + runDiffTest(t, "testdata/options", "pkg.FooEnum.txtar", WithIncludeTypes("pkg.FooEnum")) }) t.Run("service", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.FooService"}, "pkg.FooService.txtar") + runDiffTest(t, "testdata/options", "pkg.FooService.txtar", WithIncludeTypes("pkg.FooService")) }) t.Run("method", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.FooService.Do"}, "pkg.FooService.Do.txtar") + runDiffTest(t, "testdata/options", "pkg.FooService.Do.txtar", WithIncludeTypes("pkg.FooService.Do")) }) t.Run("all", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.Foo", "pkg.FooEnum", "pkg.FooService"}, "all.txtar") + runDiffTest(t, "testdata/options", "all.txtar", WithIncludeTypes("pkg.Foo", "pkg.FooEnum", "pkg.FooService")) }) t.Run("exclude-options", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.Foo", "pkg.FooEnum", "pkg.FooService"}, "all-exclude-options.txtar", WithExcludeCustomOptions()) + runDiffTest(t, "testdata/options", "all-exclude-options.txtar", WithIncludeTypes("pkg.Foo", "pkg.FooEnum", "pkg.FooService"), WithExcludeCustomOptions()) }) t.Run("files", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"Files"}, "Files.txtar") + runDiffTest(t, "testdata/options", "Files.txtar", WithIncludeTypes("Files")) }) t.Run("all-with-files", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", []string{"pkg.Foo", "pkg.FooEnum", "pkg.FooService", "Files"}, "all-with-Files.txtar") + runDiffTest(t, "testdata/options", "all-with-Files.txtar", WithIncludeTypes("pkg.Foo", "pkg.FooEnum", "pkg.FooService", "Files")) + }) + + t.Run("exclude-message", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "pkg.Foo.exclude.txtar", WithExcludeTypes("pkg.Foo")) + }) + t.Run("exclude-enum", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "pkg.FooEnum.exclude.txtar", WithExcludeTypes("pkg.FooEnum")) + }) + t.Run("exclude-service", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "pkg.FooService.exclude.txtar", WithExcludeTypes("pkg.FooService")) + }) + t.Run("exclude-method", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "pkg.FooService.Do.exclude.txtar", WithExcludeTypes("pkg.FooService.Do")) + }) + t.Run("exclude-all", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "all.exclude.txtar", WithExcludeTypes("pkg.Foo", "pkg.FooEnum", "pkg.FooService")) + }) + + t.Run("mixed-service-method", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "pkg.FooService.mixed.txtar", WithIncludeTypes("pkg.FooService"), WithExcludeTypes("pkg.FooService.Do")) }) } @@ -88,19 +114,117 @@ func TestNesting(t *testing.T) { t.Parallel() t.Run("message", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/nesting", []string{"pkg.Foo"}, "message.txtar") + runDiffTest(t, "testdata/nesting", "message.txtar", WithIncludeTypes("pkg.Foo")) }) t.Run("recursenested", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/nesting", []string{"pkg.Foo.NestedFoo.NestedNestedFoo"}, "recursenested.txtar") + runDiffTest(t, "testdata/nesting", "recursenested.txtar", WithIncludeTypes("pkg.Foo.NestedFoo.NestedNestedFoo")) }) t.Run("enum", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/nesting", []string{"pkg.FooEnum"}, "enum.txtar") + runDiffTest(t, "testdata/nesting", "enum.txtar", WithIncludeTypes("pkg.FooEnum")) }) t.Run("usingother", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/nesting", []string{"pkg.Baz"}, "usingother.txtar") + runDiffTest(t, "testdata/nesting", "usingother.txtar", WithIncludeTypes("pkg.Baz")) + }) + + t.Run("exclude_message", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/nesting", "message.exclude.txtar", WithExcludeTypes("pkg.Foo")) + }) + t.Run("exclude_recursenested", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/nesting", "recursenested.exclude.txtar", WithExcludeTypes("pkg.Foo.NestedFoo.NestedNestedFoo")) + }) + t.Run("exclude_enum", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/nesting", "enum.exclude.txtar", WithExcludeTypes("pkg.FooEnum")) + }) + t.Run("exclude_usingother", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/nesting", "usingother.exclude.txtar", WithExcludeTypes("pkg.Baz")) + }) + + t.Run("mixed", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/nesting", "mixed.txtar", WithIncludeTypes("pkg.Foo", "pkg.FooEnum"), WithExcludeTypes("pkg.Foo.NestedFoo", "pkg.Baz")) + }) +} + +func TestOptions(t *testing.T) { + t.Parallel() + t.Run("include_jstype", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "options.jstype.include.txtar", WithIncludeOptions("google.protobuf.FieldOptions.jstype")) + WithIncludeOptions("google.protobuf.FieldOptions.jstype") + }) + t.Run("exclude_jstype", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "options.jstype.exclude.txtar", WithExcludeOptions("google.protobuf.FieldOptions.jstype")) + }) + t.Run("include_message", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "options.message.include.txtar", WithIncludeOptions("message_foo", "message_baz")) + }) + t.Run("exclude_foo", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "options.foo.exclude.txtar", WithExcludeOptions( + "message_foo", + "field_foo", + "oneof_foo", + "enum_foo", + "enum_value_foo", + "service_foo", + "method_foo", + "UsedOption.file_foo", + )) + }) + t.Run("include_foo", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "options.foo.include.txtar", WithIncludeOptions( + "message_foo", + "field_foo", + "oneof_foo", + "enum_foo", + "enum_value_foo", + "service_foo", + "method_foo", + "UsedOption.file_foo", + )) + }) + t.Run("only_file", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "options.only_file.txtar", WithExcludeOptions( + "message_foo", "message_bar", "message_baz", + "field_foo", "field_bar", "field_baz", + "oneof_foo", "oneof_bar", "oneof_baz", + "enum_foo", "enum_bar", "enum_baz", + "enum_value_foo", "enum_value_bar", "enum_value_baz", + "service_foo", "service_bar", "service_baz", + "method_foo", "method_bar", "method_baz", + )) + }) +} + +func TestOptionImports(t *testing.T) { + t.Parallel() + + t.Run("none_excluded", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/imports", "none_excluded.txtar", WithExcludeOptions("google.protobuf.FieldOptions.jstype")) + }) + t.Run("exclude_foo", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/imports", "foo.txtar", WithExcludeOptions("message_foo")) + }) + t.Run("exclude_foo_bar", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/imports", "foo_bar.txtar", WithExcludeOptions("message_foo", "message_bar")) + }) + t.Run("exclude_bar", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/imports", "bar.txtar", WithIncludeOptions("message_foo"), WithExcludeOptions("message_bar")) }) } @@ -108,44 +232,44 @@ func TestImportModifiers(t *testing.T) { t.Parallel() t.Run("regular_weak", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/importmods", []string{"ImportRegular", "ImportWeak"}, "regular_weak.txtar") + runDiffTest(t, "testdata/importmods", "regular_weak.txtar", WithIncludeTypes("ImportRegular", "ImportWeak")) }) t.Run("weak_public", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/importmods", []string{"ImportWeak", "ImportPublic"}, "weak_public.txtar") + runDiffTest(t, "testdata/importmods", "weak_public.txtar", WithIncludeTypes("ImportWeak", "ImportPublic")) }) t.Run("regular_public", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/importmods", []string{"ImportRegular", "ImportPublic"}, "regular_public.txtar") + runDiffTest(t, "testdata/importmods", "regular_public.txtar", WithIncludeTypes("ImportRegular", "ImportPublic")) }) t.Run("noimports", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/importmods", []string{"NoImports"}, "noimports.txtar") + runDiffTest(t, "testdata/importmods", "noimports.txtar", WithIncludeTypes("NoImports")) }) } func TestExtensions(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/extensions", []string{"pkg.Foo"}, "extensions.txtar") - runDiffTest(t, "testdata/extensions", []string{"pkg.Foo"}, "extensions-excluded.txtar", WithExcludeKnownExtensions()) + runDiffTest(t, "testdata/extensions", "extensions.txtar", WithIncludeTypes("pkg.Foo")) + runDiffTest(t, "testdata/extensions", "extensions-excluded.txtar", WithExcludeKnownExtensions(), WithIncludeTypes("pkg.Foo")) } func TestPackages(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/packages", []string{""}, "root.txtar") - runDiffTest(t, "testdata/packages", []string{"foo"}, "foo.txtar") - runDiffTest(t, "testdata/packages", []string{"foo.bar"}, "foo.bar.txtar") - runDiffTest(t, "testdata/packages", []string{"foo.bar.baz"}, "foo.bar.baz.txtar") + runDiffTest(t, "testdata/packages", "root.txtar", WithIncludeTypes("")) + runDiffTest(t, "testdata/packages", "foo.txtar", WithIncludeTypes("foo")) + runDiffTest(t, "testdata/packages", "foo.bar.txtar", WithIncludeTypes("foo.bar")) + runDiffTest(t, "testdata/packages", "foo.bar.baz.txtar", WithIncludeTypes("foo.bar.baz")) } func TestAny(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax"}, "c1.txtar") - runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InField"}, "c2.txtar") // c2 - runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InList"}, "c3.txtar") - runDiffTest(t, "testdata/any", []string{"ExtendedAnySyntax_InMap"}, "c4.txtar") - runDiffTest(t, "testdata/any", []string{"NormalMessageSyntaxValidType"}, "d.txtar") - runDiffTest(t, "testdata/any", []string{"NormalMessageSyntaxInvalidType"}, "e.txtar") + runDiffTest(t, "testdata/any", "c1.txtar", WithIncludeTypes("ExtendedAnySyntax")) + runDiffTest(t, "testdata/any", "c2.txtar", WithIncludeTypes("ExtendedAnySyntax_InField")) // c2 + runDiffTest(t, "testdata/any", "c3.txtar", WithIncludeTypes("ExtendedAnySyntax_InList")) + runDiffTest(t, "testdata/any", "c4.txtar", WithIncludeTypes("ExtendedAnySyntax_InMap")) + runDiffTest(t, "testdata/any", "d.txtar", WithIncludeTypes("NormalMessageSyntaxValidType")) + runDiffTest(t, "testdata/any", "e.txtar", WithIncludeTypes("NormalMessageSyntaxInvalidType")) } func TestSourceCodeInfo(t *testing.T) { @@ -255,16 +379,16 @@ func getImage(ctx context.Context, logger *slog.Logger, testdataDir string, opti return bucket, image, nil } -func runDiffTest(t *testing.T, testdataDir string, typenames []string, expectedFile string, opts ...ImageFilterOption) { +func runDiffTest(t *testing.T, testdataDir string, expectedFile string, opts ...ImageFilterOption) { ctx := context.Background() bucket, image, err := getImage(ctx, slogtestext.NewLogger(t), testdataDir, bufimage.WithExcludeSourceCodeInfo()) require.NoError(t, err) _ = protojson.MarshalOptions{} - //b, _ := protojson.MarshalOptions{Indent: " "}.Marshal(bufimage.ImageToFileDescriptorSet(image)) - //t.Log(string(b)) + b, _ := protojson.MarshalOptions{Indent: " "}.Marshal(bufimage.ImageToFileDescriptorSet(image)) + t.Log(string(b)) - filteredImage, err := FilterImage(image, append(opts, WithIncludeTypes(typenames...))...) + filteredImage, err := FilterImage(image, opts...) require.NoError(t, err) assert.NotNil(t, image) assert.True(t, imageIsDependencyOrdered(filteredImage), "image files not in dependency order") diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 2ee645f6a7..b869baf9c5 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -31,25 +31,34 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im if err != nil { return nil, err } - for SOMETHING, pkg := range imageIndex.Packages { - fmt.Println("PACKAGE", SOMETHING, pkg.fullName) - } closure := newTransitiveClosure() - // TODO: handle options. - // TODO: add all excludes first. + // All excludes are added first, then includes walk included all non excluded types. + for excludeType := range options.excludeTypes { + excludeType := protoreflect.FullName(excludeType) + if err := closure.excludeType(excludeType, imageIndex, options); err != nil { + return nil, err + } + } for includeType := range options.includeTypes { includeType := protoreflect.FullName(includeType) - if err := closure.addType(includeType, imageIndex, options); err != nil { + if err := closure.includeType(includeType, imageIndex, options); err != nil { return nil, err } } + // TODO: No types were included, so include everything. This can be + // removed when we are able to handle finding all required imports + // below, when remapping the descriptor. + if len(options.includeTypes) == 0 { + for _, file := range image.Files() { + if err := closure.addElement(file.FileDescriptorProto(), "", false, imageIndex, options); err != nil { + return nil, err + } + } + } // After all types are added, add their known extensions if err := closure.addExtensions(imageIndex, options); err != nil { return nil, err } - for key, value := range closure.elements { - fmt.Println("CLOSURE", key.GetName(), value) - } // Loop over image files in revserse DAG order. Imports that are no longer // imported by a previous file are dropped from the image. @@ -60,12 +69,10 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im for i := len(image.Files()) - 1; i >= 0; i-- { imageFile := imageFiles[i] imageFilePath := imageFile.Path() - fmt.Println("IMAGE FILE", i, imageFilePath) _, isFileImported := importsByFilePath[imageFilePath] if imageFile.IsImport() && !options.allowImportedTypes { // Check if this import is still used. if !isFileImported { - fmt.Println(" NOT IMPORTED") continue } } @@ -83,7 +90,6 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im if isFileImported { return nil, fmt.Errorf("imported file %q was filtered out", imageFilePath) } - fmt.Println(" FILTERED OUT") continue // Filtered out. } for _, filePath := range newImageFile.FileDescriptorProto().Dependency { @@ -92,7 +98,6 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im newImageFiles = append(newImageFiles, newImageFile) } if dirty { - fmt.Println("Creating the new image") // Reverse the image files back to DAG order. slices.Reverse(newImageFiles) return bufimage.NewImage(newImageFiles) @@ -123,15 +128,6 @@ func filterImageFile( if newFileDescriptor == fileDescriptor { return imageFile, nil // No changes required. } - fmt.Println("-----") - //b, _ := (&protojson.MarshalOptions{ - // Indent: " ", - //}).Marshal(newFileDescriptor) - //fmt.Println(string(b)) - //fmt.Println("FILE", newFileDescriptor.GetName()) - for _, filePath := range newFileDescriptor.Dependency { - fmt.Println(" IMPORT", filePath) - } // Remap the source code info. if locations := fileDescriptor.SourceCodeInfo.GetLocation(); len(locations) > 0 { @@ -179,10 +175,7 @@ func (b *sourcePathsBuilder) remapFileDescriptor( sourcePathsRemap *sourcePathsRemapTrie, fileDescriptor *descriptorpb.FileDescriptorProto, ) (*descriptorpb.FileDescriptorProto, error) { - fmt.Println("FILE", fileDescriptor.GetName()) - packageName := protoreflect.FullName(fileDescriptor.GetPackage()) if !b.closure.hasType(fileDescriptor, b.options) { - fmt.Println(" FILTERED BY PACKAGE", packageName) return nil, nil } @@ -190,13 +183,10 @@ func (b *sourcePathsBuilder) remapFileDescriptor( // Walk the file descriptor. isDirty := false - fmt.Println("REMAP MESSAGES") newMessages, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, b.remapDescriptor) if err != nil { return nil, err } - fmt.Println("MESSAGES", len(newMessages)) - fmt.Println("MESSAGES", newMessages) isDirty = isDirty || changed newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, b.remapEnum) if err != nil { @@ -214,10 +204,6 @@ func (b *sourcePathsBuilder) remapFileDescriptor( if err != nil { return nil, err } - fmt.Println("CHANGED?", changed, "how many?", len(newExtensions)) - //if len(newExtensions) == 0 { - // sourcePathsRemap.markDeleted(append(sourcePath, fileExtensionsTag)) - //} isDirty = isDirty || changed newOptions, changed, err := remapMessage(nil, append(sourcePath, fileOptionsTag), fileDescriptor.Options, b.remapOptions) if err != nil { @@ -226,7 +212,6 @@ func (b *sourcePathsBuilder) remapFileDescriptor( isDirty = isDirty || changed if !isDirty { - fmt.Println(" NO CHANGES", fileDescriptor.GetName()) return fileDescriptor, nil } @@ -295,13 +280,11 @@ func (b *sourcePathsBuilder) remapDescriptor( descriptor *descriptorpb.DescriptorProto, ) (*descriptorpb.DescriptorProto, bool, error) { if !b.closure.hasType(descriptor, b.options) { - fmt.Println(" FILTERED BY DESCRIPTOR", descriptor.GetName()) return nil, true, nil } var newDescriptor *descriptorpb.DescriptorProto isDirty := false if mode := b.closure.elements[descriptor]; mode == inclusionModeEnclosing { - fmt.Println("--- ENCLOSING:", descriptor.GetName()) // If the type is only enclosing, only the namespace matters. isDirty = true newDescriptor = shallowClone(descriptor) @@ -345,10 +328,6 @@ func (b *sourcePathsBuilder) remapDescriptor( if err != nil { return nil, false, err } - fmt.Println("DESCRIPTOR", changed, "how many?", len(newExtensions)) - //if len(newExtensions) == 0 { - // sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionsTag)) - //} isDirty = isDirty || changed newDescriptors, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.remapDescriptor) if err != nil { @@ -367,7 +346,6 @@ func (b *sourcePathsBuilder) remapDescriptor( isDirty = isDirty || changed if !isDirty { - fmt.Println(" NO CHANGES", descriptor.GetName()) return descriptor, false, nil } if newDescriptor == nil { @@ -518,7 +496,6 @@ func (b *sourcePathsBuilder) remapField( ) (*descriptorpb.FieldDescriptorProto, bool, error) { if field.Extendee != nil { // Extensions are filtered by type. - fmt.Println("EXTEND", field.GetName()) if !b.closure.hasType(field, b.options) { return nil, true, nil } @@ -529,7 +506,6 @@ func (b *sourcePathsBuilder) remapField( descriptorpb.FieldDescriptorProto_TYPE_GROUP: typeName := protoreflect.FullName(strings.TrimPrefix(field.GetTypeName(), ".")) typeInfo := b.imageIndex.ByName[typeName] - fmt.Println("FIELD", field.GetName(), "->", typeName) if !b.closure.hasType(typeInfo.element, b.options) { return nil, true, nil } diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter_test.go b/private/bufpkg/bufimage/bufimageutil/image_filter_test.go deleted file mode 100644 index 93f5b10a1d..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/image_filter_test.go +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2020-2025 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimageutil - -/*import ( - "bytes" - "context" - "sort" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" - "github.com/bufbuild/buf/private/pkg/slicesext" - "github.com/bufbuild/buf/private/pkg/slogtestext" - "github.com/bufbuild/buf/private/pkg/storage" - "github.com/bufbuild/buf/private/pkg/storage/storageos" - "github.com/jhump/protoreflect/v2/protoprint" - "github.com/stretchr/testify/require" - "golang.org/x/tools/txtar" - "google.golang.org/protobuf/reflect/protodesc" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/types/descriptorpb" -) - -func TestExcludeOptions(t *testing.T) { - t.Parallel() - - t.Run("NoneExcluded", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", - ) - }) - t.Run("ExcludeMessage", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", WithExcludeOptions( - "message_foo", "message_bar", "message_baz", - ), - ) - }) - t.Run("ExcludeFoo", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", WithExcludeOptions( - "message_foo", - "field_foo", - "oneof_foo", - "enum_foo", - "enum_value_foo", - "service_foo", - "method_foo", - "UsedOption.file_foo", - ), - ) - }) - t.Run("OnlyFile", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", WithExcludeOptions( - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "oneof_foo", "oneof_bar", "oneof_baz", - "enum_foo", "enum_bar", "enum_baz", - "enum_value_foo", "enum_value_bar", "enum_value_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - ), - ) - }) - t.Run("OnlyOneOf", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", WithExcludeOptions( - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "enum_foo", "enum_bar", "enum_baz", - "enum_value_foo", "enum_value_bar", "enum_value_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", - ), - ) - }) - t.Run("OnlyEnumValue", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", WithExcludeOptions( - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "oneof_foo", "oneof_bar", "oneof_baz", - "enum_foo", "enum_bar", "enum_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", - ), - ) - }) - t.Run("ExcludeAll", func(t *testing.T) { - t.Parallel() - testFilterOptions( - t, "testdata/excludeoptions", WithExcludeOptions( - "message_foo", "message_bar", "message_baz", - "field_foo", "field_bar", "field_baz", - "oneof_foo", "oneof_bar", "oneof_baz", - "enum_foo", "enum_bar", "enum_baz", - "enum_value_foo", "enum_value_bar", "enum_value_baz", - "service_foo", "service_bar", "service_baz", - "method_foo", "method_bar", "method_baz", - "UsedOption.file_foo", "UsedOption.file_bar", "UsedOption.file_baz", - ), - ) - }) -} - -func TestExcludeOptionImports(t *testing.T) { - t.Parallel() - - // This checks that when excluding options the imports are correctly dropped. - // For this case when both options are removed only a.proto should be left. - testdataDir := "testdata/excludeoptionimports" - bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) - require.NoError(t, err) - testModuleData := []bufmoduletesting.ModuleData{ - { - Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("a.proto")), - }, - { - Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("options.proto")), - NotTargeted: true, - }, - } - moduleSet, err := bufmoduletesting.NewModuleSet(testModuleData...) - require.NoError(t, err) - - // Safe to filter the image concurrently as its not being modified. - image, err := bufimage.BuildImage( - context.Background(), - slogtestext.NewLogger(t), - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), - bufimage.WithExcludeSourceCodeInfo(), - ) - require.NoError(t, err) - - t.Run("NoneExcluded", func(t *testing.T) { - t.Parallel() - testFilterOptionsForImage(t, bucket, image) - }) - t.Run("ExcludeFoo", func(t *testing.T) { - t.Parallel() - testFilterOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo")) - }) - t.Run("ExcludeFooBar", func(t *testing.T) { - t.Parallel() - testFilterOptionsForImage(t, bucket, image, WithExcludeOptions("message_foo", "message_bar")) - }) - t.Run("ExcludeBar", func(t *testing.T) { - t.Parallel() - testFilterOptionsForImage(t, bucket, image, WithExcludeOptions("message_bar")) - }) -} - -func TestFilterTypes(t *testing.T) { - t.Parallel() - - testdataDir := "testdata/nesting" - bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) - require.NoError(t, err) - testModuleData := []bufmoduletesting.ModuleData{ - { - Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("a.proto")), - }, - { - Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("options.proto")), - NotTargeted: true, - }, - } - moduleSet, err := bufmoduletesting.NewModuleSet(testModuleData...) - require.NoError(t, err) - - // Safe to filter the image concurrently as its not being modified. - image, err := bufimage.BuildImage( - context.Background(), - slogtestext.NewLogger(t), - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), - bufimage.WithExcludeSourceCodeInfo(), - ) - require.NoError(t, err) - - t.Run("ExcludeBar", func(t *testing.T) { - t.Parallel() - testFilterOptionsForImage(t, bucket, image, WithExcludeTypes("pkg.Bar")) - }) - t.Run("IncludeBar", func(t *testing.T) { - t.Parallel() - testFilterOptionsForImage(t, bucket, image, WithIncludeTypes("pkg.Bar")) - }) - -} - -func testFilterOptions(t *testing.T, testdataDir string, options ...ImageFilterOption) { - bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) - require.NoError(t, err) - testFilterOptionsForModuleData(t, bucket, nil, options...) -} - -func testFilterOptionsForModuleData(t *testing.T, bucket storage.ReadWriteBucket, moduleData []bufmoduletesting.ModuleData, options ...ImageFilterOption) { - ctx := context.Background() - if len(moduleData) == 0 { - moduleData = append(moduleData, bufmoduletesting.ModuleData{ - Bucket: bucket, - }) - } - moduleSet, err := bufmoduletesting.NewModuleSet(moduleData...) - require.NoError(t, err) - - image, err := bufimage.BuildImage( - ctx, - slogtestext.NewLogger(t), - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), - bufimage.WithExcludeSourceCodeInfo(), - ) - require.NoError(t, err) - - testFilterOptionsForImage(t, bucket, image, options...) -} - -func testFilterOptionsForImage(t *testing.T, bucket storage.ReadWriteBucket, image bufimage.Image, options ...ImageFilterOption) { - ctx := context.Background() - filteredImage, err := FilterImage(image, options...) - require.NoError(t, err) - - files, err := protodesc.NewFiles(&descriptorpb.FileDescriptorSet{ - File: slicesext.Map(filteredImage.Files(), func(imageFile bufimage.ImageFile) *descriptorpb.FileDescriptorProto { - return imageFile.FileDescriptorProto() - }), - }) - require.NoError(t, err) - - archive := &txtar.Archive{} - printer := protoprint.Printer{ - SortElements: true, - Compact: true, - } - files.RangeFiles(func(fileDescriptor protoreflect.FileDescriptor) bool { - fileBuilder := &bytes.Buffer{} - require.NoError(t, printer.PrintProtoFile(fileDescriptor, fileBuilder), "expected no error while printing %q", fileDescriptor.Path()) - archive.Files = append( - archive.Files, - txtar.File{ - Name: fileDescriptor.Path(), - Data: fileBuilder.Bytes(), - }, - ) - return true - }) - sort.SliceStable(archive.Files, func(i, j int) bool { - return archive.Files[i].Name < archive.Files[j].Name - }) - generated := txtar.Format(archive) - expectedFile := t.Name() + ".txtar" - checkExpectation(t, ctx, generated, bucket, expectedFile) -}*/ diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 99cea8a303..41dbcdb1ad 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -80,8 +80,8 @@ func newImageIndexForImage(image bufimage.Image, options *imageFilterOptions) (* index := &imageIndex{ ByName: make(map[protoreflect.FullName]elementInfo), ByDescriptor: make(map[namedDescriptor]elementInfo), - FileTypes: make(map[string][]protoreflect.FullName), Packages: make(map[string]*packageInfo), + FileTypes: make(map[string][]protoreflect.FullName), } if options.includeCustomOptions { index.NameToOptions = make(map[protoreflect.FullName]map[int32]*descriptorpb.FieldDescriptorProto) From e2c5f8bcf6cf55747ec790714d2d026b8d3af0fe Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 19:38:33 +0100 Subject: [PATCH 14/80] Fix import dropping for options --- .../bufimage/bufimageutil/bufimageutil.go | 29 +++++----- .../bufimageutil/bufimageutil_test.go | 56 +++++++++++++------ .../bufimage/bufimageutil/image_filter.go | 3 + .../options/options.jstype.include.txtar | 1 - 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 6e5ab90774..f3739b1552 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -380,7 +380,7 @@ func (t *transitiveClosure) includeType( return nil } -func (t *transitiveClosure) addImport(fromPath, toPath string) { +func (t *transitiveClosure) addImport(wat namedDescriptor, fromPath, toPath string) { if fromPath == toPath { return // no need for a file to import itself } @@ -401,9 +401,8 @@ func (t *transitiveClosure) addElement( ) error { descriptorInfo := imageIndex.ByDescriptor[descriptor] if referrerFile != "" { - t.addImport(referrerFile, descriptorInfo.file.Path()) + t.addImport(descriptor, referrerFile, descriptorInfo.file.Path()) } - if existingMode, ok := t.elements[descriptor]; ok && existingMode != inclusionModeEnclosing { if existingMode == inclusionModeImplicit && !impliedByCustomOption { // upgrade from implied to explicitly part of closure @@ -417,15 +416,6 @@ func (t *transitiveClosure) addElement( t.elements[descriptor] = inclusionModeExplicit } - // if this type is enclosed inside another, add enclosing types - if err := t.addEnclosing(descriptorInfo.parent, descriptorInfo.file.Path(), imageIndex, opts); err != nil { - return err - } - // add any custom options and their dependencies - if err := t.exploreCustomOptions(descriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { - return err - } - switch typedDescriptor := descriptor.(type) { case *descriptorpb.FileDescriptorProto: typeNames, ok := imageIndex.FileTypes[typedDescriptor.GetName()] @@ -536,6 +526,15 @@ func (t *transitiveClosure) addElement( return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) } + // if this type is enclosed inside another, add enclosing types + if err := t.addEnclosing(descriptorInfo.parent, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + // add any custom options and their dependencies + if err := t.exploreCustomOptions(descriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { + return err + } + return nil } @@ -805,9 +804,9 @@ func (t *transitiveClosure) exploreCustomOptions( optionsName := options.Descriptor().FullName() var err error options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - //if !t.hasOption(fd, opts) { - // return true - //} + if !t.hasOption(fd, opts) { + return true + } // If the value contains an Any message, we should add the message type // therein to the closure. if err = t.exploreOptionValueForAny(fd, val, referrerFile, imageIndex, opts); err != nil { diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index a96841dce0..6651ded534 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -35,7 +35,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/tools/txtar" - "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" @@ -210,21 +209,51 @@ func TestOptions(t *testing.T) { func TestOptionImports(t *testing.T) { t.Parallel() + // This checks that when excluding options the imports are correctly dropped. + // For this case when both options are removed only a.proto should be left. + testdataDir := "testdata/imports" + bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) + require.NoError(t, err) + testModuleData := []bufmoduletesting.ModuleData{ + { + Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("a.proto")), + }, + { + Bucket: storage.FilterReadBucket(bucket, storage.MatchPathEqual("options.proto")), + NotTargeted: true, + }, + } + moduleSet, err := bufmoduletesting.NewModuleSet(testModuleData...) + require.NoError(t, err) + + // Safe to filter the image concurrently as its not being modified. + image, err := bufimage.BuildImage( + context.Background(), + slogtestext.NewLogger(t), + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), + bufimage.WithExcludeSourceCodeInfo(), + ) + require.NoError(t, err) + t.Run("none_excluded", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/imports", "none_excluded.txtar", WithExcludeOptions("google.protobuf.FieldOptions.jstype")) + generated := runFilterImage(t, image, WithExcludeOptions("google.protobuf.FieldOptions.jstype")) + checkExpectation(t, context.Background(), generated, bucket, "none_excluded.txtar") }) t.Run("exclude_foo", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/imports", "foo.txtar", WithExcludeOptions("message_foo")) + generated := runFilterImage(t, image, WithExcludeOptions("message_foo")) + checkExpectation(t, context.Background(), generated, bucket, "foo.txtar") }) t.Run("exclude_foo_bar", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/imports", "foo_bar.txtar", WithExcludeOptions("message_foo", "message_bar")) + generated := runFilterImage(t, image, WithExcludeOptions("message_foo", "message_bar")) + checkExpectation(t, context.Background(), generated, bucket, "foo_bar.txtar") }) t.Run("exclude_bar", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/imports", "bar.txtar", WithIncludeOptions("message_foo"), WithExcludeOptions("message_bar")) + generated := runFilterImage(t, image, WithIncludeOptions("message_foo"), WithExcludeOptions("message_bar")) + checkExpectation(t, context.Background(), generated, bucket, "bar.txtar") }) } @@ -383,11 +412,11 @@ func runDiffTest(t *testing.T, testdataDir string, expectedFile string, opts ... ctx := context.Background() bucket, image, err := getImage(ctx, slogtestext.NewLogger(t), testdataDir, bufimage.WithExcludeSourceCodeInfo()) require.NoError(t, err) + generated := runFilterImage(t, image, opts...) + checkExpectation(t, ctx, generated, bucket, expectedFile) +} - _ = protojson.MarshalOptions{} - b, _ := protojson.MarshalOptions{Indent: " "}.Marshal(bufimage.ImageToFileDescriptorSet(image)) - t.Log(string(b)) - +func runFilterImage(t *testing.T, image bufimage.Image, opts ...ImageFilterOption) []byte { filteredImage, err := FilterImage(image, opts...) require.NoError(t, err) assert.NotNil(t, image) @@ -395,8 +424,7 @@ func runDiffTest(t *testing.T, testdataDir string, expectedFile string, opts ... // We may have filtered out custom options from the set in the step above. However, the options messages // still contain extension fields that refer to the custom options, as a result of building the image. - // So we serialize and then de-serialize, and use only the filtered results to parse extensions. That - // way, the result will omit custom options that aren't present in the filtered set (as they will be + // So we serialize and then de-serialize, and use only the filtered results to parse extensions. That way, the result will omit custom options that aren't present in the filtered set (as they will be // considered unrecognized fields). data, err := protoencoding.NewWireMarshaler().Marshal(bufimage.ImageToFileDescriptorSet(filteredImage)) require.NoError(t, err) @@ -404,10 +432,6 @@ func runDiffTest(t *testing.T, testdataDir string, expectedFile string, opts ... err = protoencoding.NewWireUnmarshaler(filteredImage.Resolver()).Unmarshal(data, fileDescriptorSet) require.NoError(t, err) - t.Log("--------------------------------------------") - //b, _ = protojson.MarshalOptions{Indent: " "}.Marshal(bufimage.ImageToFileDescriptorSet(image)) - //t.Log(string(b)) - files, err := protodesc.NewFiles(fileDescriptorSet) require.NoError(t, err) @@ -432,7 +456,7 @@ func runDiffTest(t *testing.T, testdataDir string, expectedFile string, opts ... return archive.Files[i].Name < archive.Files[j].Name }) generated := txtar.Format(archive) - checkExpectation(t, ctx, generated, bucket, expectedFile) + return generated } func checkExpectation(t *testing.T, ctx context.Context, actual []byte, bucket storage.ReadWriteBucket, expectedFile string) { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index b869baf9c5..6bc6be6c3a 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -50,6 +50,9 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im // below, when remapping the descriptor. if len(options.includeTypes) == 0 { for _, file := range image.Files() { + if file.IsImport() { + continue + } if err := closure.addElement(file.FileDescriptorProto(), "", false, imageIndex, options); err != nil { return nil, err } diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar index 2b83275219..95f4d7a12b 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar @@ -1,7 +1,6 @@ -- a.proto -- syntax = "proto2"; package pkg; -import "options.proto"; message Empty { } message Foo { From 8e286f44b8776ce20d193b110f7f28f8fb5bf883 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 19:42:44 +0100 Subject: [PATCH 15/80] Fix build --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 6 ++---- private/bufpkg/bufimage/bufimageutil/image_index.go | 8 -------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 6bc6be6c3a..12828702cf 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -261,19 +261,18 @@ func (b *sourcePathsBuilder) remapFileDescriptor( if len(fileDescriptor.WeakDependency) > 0 { weakDependencyPath := append(sourcePath, fileWeakDependencyTag) for _, indexFrom := range fileDescriptor.WeakDependency { - path := append(weakDependencyPath, int32(indexFrom)) + path := append(weakDependencyPath, indexFrom) indexTo := dependencyChanges[indexFrom] if indexTo == -1 { sourcePathsRemap.markDeleted(path) } else { - if indexTo != int32(indexFrom) { + if indexTo != indexFrom { sourcePathsRemap.markMoved(path, indexTo) } newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, indexTo) } } } - return newFileDescriptor, nil } @@ -597,7 +596,6 @@ func remapMessage[T proto.Message]( } newMessage, _ := newMessageOpaque.(T) // Safe to assert. return newMessage, true, nil - } func remapSlice[T any]( diff --git a/private/bufpkg/bufimage/bufimageutil/image_index.go b/private/bufpkg/bufimage/bufimageutil/image_index.go index 41dbcdb1ad..7100183e8b 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_index.go +++ b/private/bufpkg/bufimage/bufimageutil/image_index.go @@ -197,11 +197,3 @@ func isOptionsTypeName(typeName protoreflect.FullName) bool { return false } } - -func getFullName(parentName protoreflect.FullName, descriptor namedDescriptor) protoreflect.FullName { - fullName := protoreflect.FullName(descriptor.GetName()) - if parentName == "" { - return fullName - } - return parentName + "." + fullName -} From cb5a8ee1f6b6b4e40504000885ed019c13b05a3c Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 20:50:58 +0100 Subject: [PATCH 16/80] Mutate in place --- .../bufimage/bufimageutil/bufimageutil.go | 8 +++++ .../bufimageutil/bufimageutil_test.go | 2 +- .../bufimage/bufimageutil/image_filter.go | 34 ++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index f3739b1552..d00cd68430 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -138,6 +138,13 @@ func WithIncludeOptions(typeNames ...string) ImageFilterOption { } } +// TODO: doc. +func WithMutateInPlace() ImageFilterOption { + return func(opts *imageFilterOptions) { + opts.mutateInPlace = true + } +} + // WithExcludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies // the set of options that should be excluded from the filtered image. // @@ -980,6 +987,7 @@ type imageFilterOptions struct { includeCustomOptions bool includeKnownExtensions bool allowImportedTypes bool + mutateInPlace bool includeTypes map[string]struct{} excludeTypes map[string]struct{} includeOptions map[string]struct{} diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 6651ded534..d31ce7c2da 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -634,7 +634,7 @@ func benchmarkFilterImage(b *testing.B, opts ...bufimage.BuildImageOption) { require.NoError(b, err) b.StartTimer() - _, err = FilterImage(image, WithIncludeTypes(typeName)) + _, err = FilterImage(image, WithIncludeTypes(typeName), WithMutateInPlace()) require.NoError(b, err) i++ if i == b.N { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 12828702cf..9cb0752d1e 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -142,7 +142,7 @@ func filterImageFile( continue } if noComment || !slices.Equal(oldPath, newPath) { - location = shallowClone(location) + location = maybeClone(location, options) location.Path = newPath } if noComment { @@ -218,7 +218,7 @@ func (b *sourcePathsBuilder) remapFileDescriptor( return fileDescriptor, nil } - newFileDescriptor := shallowClone(fileDescriptor) + newFileDescriptor := maybeClone(fileDescriptor, b.options) newFileDescriptor.MessageType = newMessages newFileDescriptor.EnumType = newEnums newFileDescriptor.Service = newServices @@ -289,7 +289,7 @@ func (b *sourcePathsBuilder) remapDescriptor( if mode := b.closure.elements[descriptor]; mode == inclusionModeEnclosing { // If the type is only enclosing, only the namespace matters. isDirty = true - newDescriptor = shallowClone(descriptor) + newDescriptor = maybeClone(descriptor, b.options) sourcePathsRemap.markNoComment(sourcePath) sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) @@ -318,7 +318,7 @@ func (b *sourcePathsBuilder) remapDescriptor( } isDirty = isDirty || changed if isDirty { - newDescriptor = shallowClone(descriptor) + newDescriptor = maybeClone(descriptor, b.options) newDescriptor.Field = newFields newDescriptor.OneofDecl = newOneofs newDescriptor.ExtensionRange = newExtensionRange @@ -351,7 +351,7 @@ func (b *sourcePathsBuilder) remapDescriptor( return descriptor, false, nil } if newDescriptor == nil { - newDescriptor = shallowClone(descriptor) + newDescriptor = maybeClone(descriptor, b.options) } newDescriptor.Extension = newExtensions newDescriptor.NestedType = newDescriptors @@ -372,7 +372,7 @@ func (b *sourcePathsBuilder) remapExtensionRange( if !changed { return extensionRange, false, nil } - newExtensionRange := shallowClone(extensionRange) + newExtensionRange := maybeClone(extensionRange, b.options) newExtensionRange.Options = newOptions return newExtensionRange, true, nil } @@ -402,7 +402,7 @@ func (b *sourcePathsBuilder) remapEnum( if !isDirty { return enum, true, nil } - newEnum := shallowClone(enum) + newEnum := maybeClone(enum, b.options) newEnum.Options = newOptions newEnum.Value = newEnumValues return newEnum, true, nil @@ -420,7 +420,7 @@ func (b *sourcePathsBuilder) remapEnumValue( if !changed { return enumValue, false, nil } - newEnumValue := shallowClone(enumValue) + newEnumValue := maybeClone(enumValue, b.options) newEnumValue.Options = newOptions return newEnumValue, true, nil } @@ -437,7 +437,7 @@ func (b *sourcePathsBuilder) remapOneof( if !changed { return oneof, false, nil } - newOneof := shallowClone(oneof) + newOneof := maybeClone(oneof, b.options) newOneof.Options = options return newOneof, true, nil } @@ -465,7 +465,7 @@ func (b *sourcePathsBuilder) remapService( if !isDirty { return service, false, nil } - newService := shallowClone(service) + newService := maybeClone(service, b.options) newService.Method = newMethods newService.Options = newOptions return newService, true, nil @@ -486,7 +486,7 @@ func (b *sourcePathsBuilder) remapMethod( if !changed { return method, false, nil } - newMethod := shallowClone(method) + newMethod := maybeClone(method, b.options) newMethod.Options = newOptions return newMethod, true, nil } @@ -536,7 +536,7 @@ func (b *sourcePathsBuilder) remapField( if !changed { return field, false, nil } - newField := shallowClone(field) + newField := maybeClone(field, b.options) newField.Options = newOption return newField, true, nil } @@ -558,7 +558,7 @@ func (b *sourcePathsBuilder) remapOptions( optionPath := append(optionsPath, int32(fd.Number())) sourcePathsRemap.markDeleted(optionPath) if newOptions == nil { - newOptions = shallowCloneReflect(options) + newOptions = maybeClone(optionsMessage, b.options).ProtoReflect() } newOptions.Clear(fd) return true @@ -616,6 +616,7 @@ func remapSlice[T any]( } isDirty = isDirty || changed if isDirty && newList == nil { + // TODO: filter in place... newList = make([]*T, 0, len(list)) newList = append(newList, list[:toIndex]...) } @@ -642,6 +643,13 @@ func remapSlice[T any]( return list, false, nil } +func maybeClone[T proto.Message](value T, options *imageFilterOptions) T { + if options == nil || !options.mutateInPlace { + return shallowClone(value) + } + return value +} + func shallowClone[T proto.Message](src T) T { value, _ := shallowCloneReflect(src.ProtoReflect()).Interface().(T) // Safe to assert. return value From 00ebe13d06e99584bcc92fe9a9bcfc44416c3b7d Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Feb 2025 21:14:49 +0100 Subject: [PATCH 17/80] Add mutate in place option --- .../bufimage/bufimageutil/bufimageutil.go | 23 ++++++---- .../bufimageutil/bufimageutil_test.go | 3 +- .../bufimage/bufimageutil/image_filter.go | 45 ++++++++++--------- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index d00cd68430..e7f7524e85 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -138,13 +138,6 @@ func WithIncludeOptions(typeNames ...string) ImageFilterOption { } } -// TODO: doc. -func WithMutateInPlace() ImageFilterOption { - return func(opts *imageFilterOptions) { - opts.mutateInPlace = true - } -} - // WithExcludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies // the set of options that should be excluded from the filtered image. // @@ -160,6 +153,16 @@ func WithExcludeOptions(typeNames ...string) ImageFilterOption { } } +// WithMutateInPlace returns an option for ImageFilteredByTypesWithOptions that specifies +// that the filtered image should be mutated in place. This option is useful when the +// unfiltered image is no longer needed and the caller wants to avoid the overhead of +// copying the image. +func WithMutateInPlace() ImageFilterOption { + return func(opts *imageFilterOptions) { + opts.mutateInPlace = true + } +} + // FilterImage returns a minimal image containing only the descriptors // required to define the set of types provided by the filter options. If no // filter options are provided, the original image is returned. @@ -174,8 +177,10 @@ func WithExcludeOptions(typeNames ...string) ImageFilterOption { // be altered to remove the dependency. // // This returns a new [bufimage.Image] that is a shallow copy of the underlying -// [descriptorpb.FileDescriptorProto]s of the original. The new image may therefore -// share state with the original image, so it should not be modified. +// [descriptorpb.FileDescriptorProto]s of the original. The new image may +// therefore share state with the original image, so it should not be modified. +// If the original image is no longer needed, it should be discarded. To avoid +// this sharing, use the [WithMutateInPlace] option. // // A descriptor is said to require another descriptor if the dependent // descriptor is needed to accurately and completely describe that descriptor. diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index d31ce7c2da..75503ab50b 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -26,6 +26,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" + "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/protoencoding" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/storage" @@ -610,7 +611,7 @@ func benchmarkFilterImage(b *testing.B, opts ...bufimage.BuildImageOption) { } ctx := context.Background() for _, benchmarkCase := range benchmarkCases { - _, image, err := getImage(ctx, slogtestext.NewLogger(b), benchmarkCase.folder, opts...) + _, image, err := getImage(ctx, slogtestext.NewLogger(b, slogtestext.WithLogLevel(appext.LogLevelError)), benchmarkCase.folder, opts...) require.NoError(b, err) benchmarkCase.image = image } diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 9cb0752d1e..01b4d40f43 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -186,24 +186,24 @@ func (b *sourcePathsBuilder) remapFileDescriptor( // Walk the file descriptor. isDirty := false - newMessages, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, b.remapDescriptor) + newMessages, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, b.remapDescriptor, b.options) if err != nil { return nil, err } isDirty = isDirty || changed - newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, b.remapEnum) + newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, b.remapEnum, b.options) if err != nil { return nil, err } isDirty = isDirty || changed - newServices, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileServicesTag), fileDescriptor.Service, b.remapService) + newServices, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileServicesTag), fileDescriptor.Service, b.remapService, b.options) if err != nil { return nil, err } isDirty = isDirty || changed // TODO: extension docs //newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField) - newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField) + newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField, b.options) if err != nil { return nil, err } @@ -302,17 +302,17 @@ func (b *sourcePathsBuilder) remapDescriptor( newDescriptor.ReservedRange = nil newDescriptor.ReservedName = nil } else { - newFields, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.remapField) + newFields, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.remapField, b.options) if err != nil { return nil, false, err } isDirty = isDirty || changed - newOneofs, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.remapOneof) + newOneofs, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageOneofsTag), descriptor.OneofDecl, b.remapOneof, b.options) if err != nil { return nil, false, err } isDirty = isDirty || changed - newExtensionRange, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionRangesTag), descriptor.ExtensionRange, b.remapExtensionRange) + newExtensionRange, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionRangesTag), descriptor.ExtensionRange, b.remapExtensionRange, b.options) if err != nil { return nil, false, err } @@ -326,17 +326,17 @@ func (b *sourcePathsBuilder) remapDescriptor( } // TODO: sourcePath might not be correct here. // TODO: extension docs. - newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.remapField) + newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.remapField, b.options) if err != nil { return nil, false, err } isDirty = isDirty || changed - newDescriptors, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.remapDescriptor) + newDescriptors, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageNestedMessagesTag), descriptor.NestedType, b.remapDescriptor, b.options) if err != nil { return nil, false, err } isDirty = isDirty || changed - newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.remapEnum) + newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageEnumsTag), descriptor.EnumType, b.remapEnum, b.options) if err != nil { return nil, false, err } @@ -394,7 +394,7 @@ func (b *sourcePathsBuilder) remapEnum( isDirty = changed // Walk the enum values. - newEnumValues, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, enumValuesTag), enum.Value, b.remapEnumValue) + newEnumValues, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, enumValuesTag), enum.Value, b.remapEnumValue, b.options) if err != nil { return nil, false, err } @@ -452,7 +452,7 @@ func (b *sourcePathsBuilder) remapService( } isDirty := false // Walk the service methods. - newMethods, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, serviceMethodsTag), service.Method, b.remapMethod) + newMethods, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, serviceMethodsTag), service.Method, b.remapMethod, b.options) if err != nil { return nil, false, err } @@ -603,6 +603,7 @@ func remapSlice[T any]( sourcePath protoreflect.SourcePath, list []*T, remapItem func(*sourcePathsRemapTrie, protoreflect.SourcePath, *T) (*T, bool, error), + options *imageFilterOptions, ) ([]*T, bool, error) { isDirty := false var newList []*T @@ -616,9 +617,12 @@ func remapSlice[T any]( } isDirty = isDirty || changed if isDirty && newList == nil { - // TODO: filter in place... - newList = make([]*T, 0, len(list)) - newList = append(newList, list[:toIndex]...) + if options.mutateInPlace { + newList = list[:toIndex:cap(list)] + } else { + newList = make([]*T, 0, len(list)) + newList = append(newList, list[:toIndex]...) + } } isIncluded := newItem != nil if isIncluded { @@ -650,16 +654,13 @@ func maybeClone[T proto.Message](value T, options *imageFilterOptions) T { return value } -func shallowClone[T proto.Message](src T) T { - value, _ := shallowCloneReflect(src.ProtoReflect()).Interface().(T) // Safe to assert. - return value -} - -func shallowCloneReflect(src protoreflect.Message) protoreflect.Message { +func shallowClone[T proto.Message](message T) T { + src := message.ProtoReflect() dst := src.New() src.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { dst.Set(fd, v) return true }) - return dst + value, _ := dst.Interface().(T) // Safe to assert. + return value } From 20b720b3f93257ccf6f2287d7e31122c07f0efff Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 28 Feb 2025 19:21:07 +0100 Subject: [PATCH 18/80] Update docs --- private/buf/bufctl/controller.go | 6 +++- private/buf/bufgen/generator.go | 5 +++- .../buf/cmd/buf/command/convert/convert.go | 12 ++++++-- .../bufimage/bufimageutil/bufimageutil.go | 28 +++++++++++++++---- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 617b2e9c31..32b9e32770 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -1332,7 +1332,11 @@ func filterImage( newImage = bufimage.ImageWithoutImports(newImage) } if len(functionOptions.imageTypes) > 0 { - newImage, err = bufimageutil.FilterImage(newImage, bufimageutil.WithIncludeTypes(functionOptions.imageTypes...)) + newImage, err = bufimageutil.FilterImage( + newImage, + bufimageutil.WithIncludeTypes(functionOptions.imageTypes...), + bufimageutil.WithMutateInPlace(), + ) if err != nil { return nil, err } diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 878a55d083..9a60f3e5af 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -320,7 +320,10 @@ func (g *generator) execLocalPlugin( } if len(excludeOptions) > 0 { for i, pluginImage := range pluginImages { - pluginImage, err := bufimageutil.FilterImage(pluginImage, bufimageutil.WithExcludeOptions(excludeOptions...)) + pluginImage, err := bufimageutil.FilterImage( + pluginImage, + bufimageutil.WithExcludeOptions(excludeOptions...), + ) if err != nil { return nil, err } diff --git a/private/buf/cmd/buf/command/convert/convert.go b/private/buf/cmd/buf/command/convert/convert.go index 0bd5bf5d94..aec86d1912 100644 --- a/private/buf/cmd/buf/command/convert/convert.go +++ b/private/buf/cmd/buf/command/convert/convert.go @@ -185,7 +185,11 @@ func run( resolveWellKnownType = true } if schemaImage != nil { - _, filterErr := bufimageutil.FilterImage(schemaImage, bufimageutil.WithIncludeTypes(flags.Type)) + _, filterErr := bufimageutil.FilterImage( + schemaImage, + bufimageutil.WithIncludeTypes(flags.Type), + bufimageutil.WithMutateInPlace(), + ) if errors.Is(filterErr, bufimageutil.ErrImageFilterTypeNotFound) { resolveWellKnownType = true } @@ -283,5 +287,9 @@ func wellKnownTypeImage( if err != nil { return nil, err } - return bufimageutil.FilterImage(image, bufimageutil.WithIncludeTypes(wellKnownTypeName)) + return bufimageutil.FilterImage( + image, + bufimageutil.WithIncludeTypes(wellKnownTypeName), + bufimageutil.WithMutateInPlace(), + ) } diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index e7f7524e85..5c7a9b5223 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -101,6 +101,11 @@ func WithAllowFilterByImportedType() ImageFilterOption { // WithIncludeTypes returns an option for ImageFilteredByTypesWithOptions that specifies // the set of types that should be included in the filtered image. +// +// May be provided multiple times. The type names should be fully qualified. +// For example, "google.protobuf.Any" or "buf.validate". Type or package names +// are accepted. If the type does not exist in the image, an error +// [ErrImageFilterTypeNotFound] will be returned. func WithIncludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.includeTypes == nil { @@ -114,6 +119,11 @@ func WithIncludeTypes(typeNames ...string) ImageFilterOption { // WithExcludeTypes returns an option for ImageFilteredByTypesWithOptions that specifies // the set of types that should be excluded from the filtered image. +// +// May be provided multiple times. The type names should be fully qualified. +// For example, "google.protobuf.Any" or "buf.validate". Type or package names +// are accepted. If the type does not exist in the image, an error +// [ErrImageFilterTypeNotFound] will be returned. func WithExcludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.excludeTypes == nil { @@ -127,6 +137,10 @@ func WithExcludeTypes(typeNames ...string) ImageFilterOption { // WithIncludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies // the set of options that should be included in the filtered image. +// +// May be provided multiple times. The option names should be fully qualified. +// For example, "google.protobuf.FieldOptions.jstype" or "buf.validate.field". +// If the option does not exist in the image, it will be ignored. func WithIncludeOptions(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.includeOptions == nil { @@ -141,7 +155,9 @@ func WithIncludeOptions(typeNames ...string) ImageFilterOption { // WithExcludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies // the set of options that should be excluded from the filtered image. // -// May be provided multiple times. +// May be provided multiple times. The option names should be fully qualified. +// For example, "google.protobuf.FieldOptions.jstype" or "buf.validate.field". +// If the option does not exist in the image, it will be ignored. func WithExcludeOptions(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if opts.excludeOptions == nil { @@ -339,13 +355,13 @@ func (t *transitiveClosure) hasOption( return false } } - if fieldDescriptor.IsExtension() && !options.includeCustomOptions { - return false - } if options.includeOptions != nil { _, isIncluded = options.includeOptions[string(fullName)] return isIncluded } + if fieldDescriptor.IsExtension() && !options.includeCustomOptions { + return false + } return true } @@ -392,7 +408,7 @@ func (t *transitiveClosure) includeType( return nil } -func (t *transitiveClosure) addImport(wat namedDescriptor, fromPath, toPath string) { +func (t *transitiveClosure) addImport(fromPath, toPath string) { if fromPath == toPath { return // no need for a file to import itself } @@ -413,7 +429,7 @@ func (t *transitiveClosure) addElement( ) error { descriptorInfo := imageIndex.ByDescriptor[descriptor] if referrerFile != "" { - t.addImport(descriptor, referrerFile, descriptorInfo.file.Path()) + t.addImport(referrerFile, descriptorInfo.file.Path()) } if existingMode, ok := t.elements[descriptor]; ok && existingMode != inclusionModeEnclosing { if existingMode == inclusionModeImplicit && !impliedByCustomOption { From 3cec9b8b3ec459044d0a2e0f8a3b451437fa22df Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 3 Mar 2025 16:26:32 +0100 Subject: [PATCH 19/80] Fix generation per image --- private/buf/bufgen/generator.go | 186 +++++++++++++++------------ private/buf/bufgen/image_provider.go | 60 --------- private/pkg/slicesext/slicesext.go | 5 + 3 files changed, 106 insertions(+), 145 deletions(-) delete mode 100644 private/buf/bufgen/image_provider.go diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 9a60f3e5af..9334df94a9 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -16,10 +16,12 @@ package bufgen import ( "context" + "encoding/gob" "errors" "fmt" "log/slog" "path/filepath" + "strings" connect "connectrpc.com/connect" "github.com/bufbuild/buf/private/buf/bufprotopluginexec" @@ -197,6 +199,30 @@ func (g *generator) generateCode( return nil } +// hashPluginConfigForImage returns a hash of the plugin config for the image. +// This is used to batch plugins that have the same configuration. +// The hash is based on the following properties: +// - Strategy +// - ExcludeOptions +// - RemoteHost +func hashPluginConfigForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { + type imagePluginConfigKey struct { + Strategy Strategy + ExcludeOptions []string + RemoteHost string + } + key := &imagePluginConfigKey{ + Strategy: Strategy(pluginConfig.Strategy()), + ExcludeOptions: pluginConfig.ExcludeOptions(), + RemoteHost: pluginConfig.RemoteHost(), + } + var str strings.Builder + if err := gob.NewEncoder(&str).Encode(key); err != nil { + return "", err + } + return str.String(), nil +} + func (g *generator) execPlugins( ctx context.Context, container app.EnvStdioContainer, @@ -205,28 +231,71 @@ func (g *generator) execPlugins( includeImportsOverride *bool, includeWellKnownTypesOverride *bool, ) ([]*pluginpb.CodeGeneratorResponse, error) { - imageProvider := newImageProvider(image) // Collect all of the plugin jobs so that they can be executed in parallel. jobs := make([]func(context.Context) error, 0, len(pluginConfigs)) responses := make([]*pluginpb.CodeGeneratorResponse, len(pluginConfigs)) requiredFeatures := computeRequiredFeatures(image) - remotePluginConfigTable := make(map[string][]*remotePluginExecArgs, len(pluginConfigs)) - for i, pluginConfig := range pluginConfigs { - index := i - currentPluginConfig := pluginConfig - // We're using this as a proxy for Type() == PluginConfigTypeRemote. - // - // We should be using the enum here. - remote := currentPluginConfig.RemoteHost() - if remote != "" { - remotePluginConfigTable[remote] = append( - remotePluginConfigTable[remote], - &remotePluginExecArgs{ - Index: index, - PluginConfig: currentPluginConfig, - }, + + pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, hashPluginConfigForImage) + if err != nil { + return nil, err + } + + for _, indexedPluginConfigs := range pluginConfigsForImage { + image := image + currentPluginConfig := indexedPluginConfigs[0].Value + + // Apply per-config filters. + if excludeOptions := currentPluginConfig.ExcludeOptions(); len(excludeOptions) > 0 { + var err error + image, err = bufimageutil.FilterImage( + image, + bufimageutil.WithExcludeOptions(excludeOptions...), ) - } else { + if err != nil { + return nil, err + } + } + + // Batch for each remote. + if remote := currentPluginConfig.RemoteHost(); remote != "" { + jobs = append(jobs, func(ctx context.Context) error { + results, err := g.execRemotePluginsV2( + ctx, + container, + image, + remote, + indexedPluginConfigs, + includeImportsOverride, + includeWellKnownTypesOverride, + ) + if err != nil { + return err + } + for _, result := range results { + responses[result.Index] = result.Value + } + return nil + + }) + continue + } + + // Local plugins. + var images []bufimage.Image + switch Strategy(currentPluginConfig.Strategy()) { + case StrategyAll: + images = []bufimage.Image{image} + case StrategyDirectory: + var err error + images, err = bufimage.ImageByDir(image) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown strategy: %v", currentPluginConfig.Strategy()) + } + for _, indexedPluginConfig := range indexedPluginConfigs { jobs = append(jobs, func(ctx context.Context) error { includeImports := currentPluginConfig.IncludeImports() if includeImportsOverride != nil { @@ -236,43 +305,18 @@ func (g *generator) execPlugins( if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } - excludeOptions := currentPluginConfig.ExcludeOptions() response, err := g.execLocalPlugin( ctx, container, - imageProvider, - currentPluginConfig, + images, + indexedPluginConfig.Value, includeImports, includeWellKnownTypes, - excludeOptions, - ) - if err != nil { - return err - } - responses[index] = response - return nil - }) - } - } - // Batch for each remote. - for remote, indexedPluginConfigs := range remotePluginConfigTable { - if len(indexedPluginConfigs) > 0 { - jobs = append(jobs, func(ctx context.Context) error { - results, err := g.execRemotePluginsV2( - ctx, - container, - image, - remote, - indexedPluginConfigs, - includeImportsOverride, - includeWellKnownTypesOverride, ) if err != nil { return err } - for _, result := range results { - responses[result.Index] = result.CodeGeneratorResponse - } + responses[indexedPluginConfig.Index] = response return nil }) } @@ -308,28 +352,11 @@ func (g *generator) execPlugins( func (g *generator) execLocalPlugin( ctx context.Context, container app.EnvStdioContainer, - imageProvider *imageProvider, + pluginImages []bufimage.Image, pluginConfig bufconfig.GeneratePluginConfig, includeImports bool, includeWellKnownTypes bool, - excludeOptions []string, ) (*pluginpb.CodeGeneratorResponse, error) { - pluginImages, err := imageProvider.GetImages(Strategy(pluginConfig.Strategy())) - if err != nil { - return nil, err - } - if len(excludeOptions) > 0 { - for i, pluginImage := range pluginImages { - pluginImage, err := bufimageutil.FilterImage( - pluginImage, - bufimageutil.WithExcludeOptions(excludeOptions...), - ) - if err != nil { - return nil, err - } - pluginImages[i] = pluginImage - } - } requests, err := bufimage.ImagesToCodeGeneratorRequests( pluginImages, pluginConfig.Opt(), @@ -354,38 +381,27 @@ func (g *generator) execLocalPlugin( return response, nil } -type remotePluginExecArgs struct { - Index int - PluginConfig bufconfig.GeneratePluginConfig -} - -type remotePluginExecutionResult struct { - CodeGeneratorResponse *pluginpb.CodeGeneratorResponse - Index int -} - func (g *generator) execRemotePluginsV2( ctx context.Context, container app.EnvStdioContainer, image bufimage.Image, remote string, - pluginConfigs []*remotePluginExecArgs, + indexedPluginConfigs []slicesext.Indexed[bufconfig.GeneratePluginConfig], includeImportsOverride *bool, includeWellKnownTypesOverride *bool, - // TODO: add support for exclude_options. -) ([]*remotePluginExecutionResult, error) { - requests := make([]*registryv1alpha1.PluginGenerationRequest, len(pluginConfigs)) - for i, pluginConfig := range pluginConfigs { - includeImports := pluginConfig.PluginConfig.IncludeImports() +) ([]slicesext.Indexed[*pluginpb.CodeGeneratorResponse], error) { + requests := make([]*registryv1alpha1.PluginGenerationRequest, len(indexedPluginConfigs)) + for i, pluginConfig := range indexedPluginConfigs { + includeImports := pluginConfig.Value.IncludeImports() if includeImportsOverride != nil { includeImports = *includeImportsOverride } - includeWellKnownTypes := pluginConfig.PluginConfig.IncludeWKT() + includeWellKnownTypes := pluginConfig.Value.IncludeWKT() if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } request, err := getPluginGenerationRequest( - pluginConfig.PluginConfig, + pluginConfig.Value, includeImports, includeWellKnownTypes, ) @@ -415,15 +431,15 @@ func (g *generator) execRemotePluginsV2( if len(responses) != len(requests) { return nil, fmt.Errorf("unexpected number of responses received, got %d, wanted %d", len(responses), len(requests)) } - result := make([]*remotePluginExecutionResult, 0, len(responses)) + result := make([]slicesext.Indexed[*pluginpb.CodeGeneratorResponse], 0, len(responses)) for i := range requests { codeGeneratorResponse := responses[i].GetResponse() if codeGeneratorResponse == nil { return nil, errors.New("expected code generator response") } - result = append(result, &remotePluginExecutionResult{ - CodeGeneratorResponse: codeGeneratorResponse, - Index: pluginConfigs[i].Index, + result = append(result, slicesext.Indexed[*pluginpb.CodeGeneratorResponse]{ + Value: codeGeneratorResponse, + Index: indexedPluginConfigs[i].Index, }) } return result, nil diff --git a/private/buf/bufgen/image_provider.go b/private/buf/bufgen/image_provider.go deleted file mode 100644 index 171d7058c8..0000000000 --- a/private/buf/bufgen/image_provider.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2020-2025 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufgen - -import ( - "fmt" - "sync" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" -) - -// imageProvider is used to provide the images used -// when generating with a local plugin. Each plugin is -// in control of its own Strategy - we cache the -// imagesByDir so that we only have to build it once for -// all of the plugins that configure the Directory -// strategy. -type imageProvider struct { - image bufimage.Image - imagesByDir []bufimage.Image - lock sync.Mutex -} - -func newImageProvider(image bufimage.Image) *imageProvider { - return &imageProvider{ - image: image, - } -} - -func (p *imageProvider) GetImages(strategy Strategy) ([]bufimage.Image, error) { - switch strategy { - case StrategyAll: - return []bufimage.Image{p.image}, nil - case StrategyDirectory: - p.lock.Lock() - defer p.lock.Unlock() - if p.imagesByDir == nil { - var err error - p.imagesByDir, err = bufimage.ImageByDir(p.image) - if err != nil { - return nil, err - } - } - return p.imagesByDir, nil - default: - return nil, fmt.Errorf("unknown strategy: %v", strategy) - } -} diff --git a/private/pkg/slicesext/slicesext.go b/private/pkg/slicesext/slicesext.go index 2461affd33..5d03284e52 100644 --- a/private/pkg/slicesext/slicesext.go +++ b/private/pkg/slicesext/slicesext.go @@ -317,6 +317,11 @@ func ToIndexedValuesMap[K comparable, V any](values []V, f func(V) K) map[K][]In return ToValuesMap(ToIndexed(values), func(indexedV Indexed[V]) K { return f(indexedV.Value) }) } +// ToIndexedValuesMapError calls ToValuesMapError on the indexed values. +func ToIndexedValuesMapError[K comparable, V any](values []V, f func(V) (K, error)) (map[K][]Indexed[V], error) { + return ToValuesMapError(ToIndexed(values), func(indexedV Indexed[V]) (K, error) { return f(indexedV.Value) }) +} + // ToUniqueIndexedValuesMap calls ToUniqueValuesMap on the indexed values. func ToUniqueIndexedValuesMap[K comparable, V any](values []V, f func(V) K) (map[K]Indexed[V], error) { return ToUniqueValuesMap(ToIndexed(values), func(indexedV Indexed[V]) K { return f(indexedV.Value) }) From 5997223caa5df868f36b34b6976fdbff22a735dd Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 3 Mar 2025 16:26:46 +0100 Subject: [PATCH 20/80] Cleanup --- .../TestExcludeOptionImports/ExcludeBar.txtar | 485 ----------------- .../TestExcludeOptionImports/ExcludeFoo.txtar | 485 ----------------- .../ExcludeFooBar.txtar | 9 - .../NoneExcluded.txtar | 486 ------------------ 4 files changed, 1465 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeBar.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFoo.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFooBar.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/NoneExcluded.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeBar.txtar deleted file mode 100644 index 88e0fe3290..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeBar.txtar +++ /dev/null @@ -1,485 +0,0 @@ --- a.proto -- -syntax = "proto3"; -package pkg; -import "options.proto"; -message Foo { - option (message_foo) = { foo: "str" }; - Bar nested_bar = 2; - message Bar { - string bar = 1; - } -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -option cc_enable_arenas = true; -option csharp_namespace = "Google.Protobuf.Reflection"; -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_outer_classname = "DescriptorProtos"; -option java_package = "com.google.protobuf"; -option objc_class_prefix = "GPB"; -option optimize_for = SPEED; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; - optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1 [ - edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, - edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, - edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional EnumType enum_type = 2 [ - edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, - edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - optional RepeatedFieldEncoding repeated_field_encoding = 3 [ - edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, - edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional Utf8Validation utf8_validation = 4 [ - edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, - edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional MessageEncoding message_encoding = 5 [ - edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional JsonFormat json_format = 6 [ - edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, - edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_MESSAGE, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994 [ - declaration = { - number: 1000, - full_name: ".pb.cpp", - type: ".pb.CppFeatures" - }, - declaration = { - number: 1001, - full_name: ".pb.java", - type: ".pb.JavaFeatures" - }, - declaration = { - number: 1002, - full_name: ".pb.go", - type: ".pb.GoFeatures" - }, - declaration = { - number: 9990, - full_name: ".pb.proto1", - type: ".pb.Proto1Features" - } - ]; - extensions 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", - type: ".buf.descriptor.v1.FileDescriptorSetExtension" - } - ]; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20 [deprecated = true]; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1 [packed = true]; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1 [packed = true]; - repeated int32 span = 2 [packed = true]; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_source_code_info_extension", - type: ".buf.descriptor.v1.SourceCodeInfoExtension" - } - ]; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message OptionBar { - string bar = 1; -} -message OptionFoo { - string foo = 1; -} -extend google.protobuf.MessageOptions { - OptionFoo message_foo = 50000; - OptionBar message_bar = 50001; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFoo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFoo.txtar deleted file mode 100644 index 0706f98c00..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFoo.txtar +++ /dev/null @@ -1,485 +0,0 @@ --- a.proto -- -syntax = "proto3"; -package pkg; -import "options.proto"; -message Foo { - Bar nested_bar = 2; - message Bar { - option (message_bar) = { bar: "str" }; - string bar = 1; - } -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -option cc_enable_arenas = true; -option csharp_namespace = "Google.Protobuf.Reflection"; -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_outer_classname = "DescriptorProtos"; -option java_package = "com.google.protobuf"; -option objc_class_prefix = "GPB"; -option optimize_for = SPEED; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; - optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1 [ - edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, - edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, - edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional EnumType enum_type = 2 [ - edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, - edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - optional RepeatedFieldEncoding repeated_field_encoding = 3 [ - edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, - edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional Utf8Validation utf8_validation = 4 [ - edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, - edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional MessageEncoding message_encoding = 5 [ - edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional JsonFormat json_format = 6 [ - edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, - edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_MESSAGE, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994 [ - declaration = { - number: 1000, - full_name: ".pb.cpp", - type: ".pb.CppFeatures" - }, - declaration = { - number: 1001, - full_name: ".pb.java", - type: ".pb.JavaFeatures" - }, - declaration = { - number: 1002, - full_name: ".pb.go", - type: ".pb.GoFeatures" - }, - declaration = { - number: 9990, - full_name: ".pb.proto1", - type: ".pb.Proto1Features" - } - ]; - extensions 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", - type: ".buf.descriptor.v1.FileDescriptorSetExtension" - } - ]; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20 [deprecated = true]; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1 [packed = true]; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1 [packed = true]; - repeated int32 span = 2 [packed = true]; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_source_code_info_extension", - type: ".buf.descriptor.v1.SourceCodeInfoExtension" - } - ]; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message OptionBar { - string bar = 1; -} -message OptionFoo { - string foo = 1; -} -extend google.protobuf.MessageOptions { - OptionFoo message_foo = 50000; - OptionBar message_bar = 50001; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFooBar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFooBar.txtar deleted file mode 100644 index 8209070bbc..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/ExcludeFooBar.txtar +++ /dev/null @@ -1,9 +0,0 @@ --- a.proto -- -syntax = "proto3"; -package pkg; -message Foo { - Bar nested_bar = 2; - message Bar { - string bar = 1; - } -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/NoneExcluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/NoneExcluded.txtar deleted file mode 100644 index 1003545b2f..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/TestExcludeOptionImports/NoneExcluded.txtar +++ /dev/null @@ -1,486 +0,0 @@ --- a.proto -- -syntax = "proto3"; -package pkg; -import "options.proto"; -message Foo { - option (message_foo) = { foo: "str" }; - Bar nested_bar = 2; - message Bar { - option (message_bar) = { bar: "str" }; - string bar = 1; - } -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -option cc_enable_arenas = true; -option csharp_namespace = "Google.Protobuf.Reflection"; -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_outer_classname = "DescriptorProtos"; -option java_package = "com.google.protobuf"; -option objc_class_prefix = "GPB"; -option optimize_for = SPEED; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; - optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1 [ - edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, - edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, - edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional EnumType enum_type = 2 [ - edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, - edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - optional RepeatedFieldEncoding repeated_field_encoding = 3 [ - edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, - edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional Utf8Validation utf8_validation = 4 [ - edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, - edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional MessageEncoding message_encoding = 5 [ - edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional JsonFormat json_format = 6 [ - edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, - edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_MESSAGE, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994 [ - declaration = { - number: 1000, - full_name: ".pb.cpp", - type: ".pb.CppFeatures" - }, - declaration = { - number: 1001, - full_name: ".pb.java", - type: ".pb.JavaFeatures" - }, - declaration = { - number: 1002, - full_name: ".pb.go", - type: ".pb.GoFeatures" - }, - declaration = { - number: 9990, - full_name: ".pb.proto1", - type: ".pb.Proto1Features" - } - ]; - extensions 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", - type: ".buf.descriptor.v1.FileDescriptorSetExtension" - } - ]; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20 [deprecated = true]; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1 [packed = true]; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1 [packed = true]; - repeated int32 span = 2 [packed = true]; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_source_code_info_extension", - type: ".buf.descriptor.v1.SourceCodeInfoExtension" - } - ]; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message OptionBar { - string bar = 1; -} -message OptionFoo { - string foo = 1; -} -extend google.protobuf.MessageOptions { - OptionFoo message_foo = 50000; - OptionBar message_bar = 50001; -} From 764626af40becef089e33a31b10d7a537a8c4c93 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 3 Mar 2025 17:05:53 +0100 Subject: [PATCH 21/80] Cleanup --- private/buf/bufgen/generator.go | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 9334df94a9..0d22d4a38a 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -16,12 +16,11 @@ package bufgen import ( "context" - "encoding/gob" + "encoding/json" "errors" "fmt" "log/slog" "path/filepath" - "strings" connect "connectrpc.com/connect" "github.com/bufbuild/buf/private/buf/bufprotopluginexec" @@ -202,25 +201,25 @@ func (g *generator) generateCode( // hashPluginConfigForImage returns a hash of the plugin config for the image. // This is used to batch plugins that have the same configuration. // The hash is based on the following properties: -// - Strategy // - ExcludeOptions +// - Strategy // - RemoteHost func hashPluginConfigForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { type imagePluginConfigKey struct { - Strategy Strategy ExcludeOptions []string + Strategy Strategy RemoteHost string } key := &imagePluginConfigKey{ - Strategy: Strategy(pluginConfig.Strategy()), ExcludeOptions: pluginConfig.ExcludeOptions(), + Strategy: Strategy(pluginConfig.Strategy()), RemoteHost: pluginConfig.RemoteHost(), } - var str strings.Builder - if err := gob.NewEncoder(&str).Encode(key); err != nil { + bytes, err := json.Marshal(key) + if err != nil { return "", err } - return str.String(), nil + return string(bytes), nil } func (g *generator) execPlugins( @@ -236,17 +235,18 @@ func (g *generator) execPlugins( responses := make([]*pluginpb.CodeGeneratorResponse, len(pluginConfigs)) requiredFeatures := computeRequiredFeatures(image) + // Group the pluginConfigs by their hash. The hash only considers the + // properties ExcludeOptions, Strategy, and RemoteHost. pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, hashPluginConfigForImage) if err != nil { return nil, err } - for _, indexedPluginConfigs := range pluginConfigsForImage { image := image - currentPluginConfig := indexedPluginConfigs[0].Value + hashPluginConfig := indexedPluginConfigs[0].Value // Apply per-config filters. - if excludeOptions := currentPluginConfig.ExcludeOptions(); len(excludeOptions) > 0 { + if excludeOptions := hashPluginConfig.ExcludeOptions(); len(excludeOptions) > 0 { var err error image, err = bufimageutil.FilterImage( image, @@ -258,7 +258,7 @@ func (g *generator) execPlugins( } // Batch for each remote. - if remote := currentPluginConfig.RemoteHost(); remote != "" { + if remote := hashPluginConfig.RemoteHost(); remote != "" { jobs = append(jobs, func(ctx context.Context) error { results, err := g.execRemotePluginsV2( ctx, @@ -276,14 +276,13 @@ func (g *generator) execPlugins( responses[result.Index] = result.Value } return nil - }) continue } // Local plugins. var images []bufimage.Image - switch Strategy(currentPluginConfig.Strategy()) { + switch Strategy(hashPluginConfig.Strategy()) { case StrategyAll: images = []bufimage.Image{image} case StrategyDirectory: @@ -293,15 +292,15 @@ func (g *generator) execPlugins( return nil, err } default: - return nil, fmt.Errorf("unknown strategy: %v", currentPluginConfig.Strategy()) + return nil, fmt.Errorf("unknown strategy: %v", hashPluginConfig.Strategy()) } for _, indexedPluginConfig := range indexedPluginConfigs { jobs = append(jobs, func(ctx context.Context) error { - includeImports := currentPluginConfig.IncludeImports() + includeImports := hashPluginConfig.IncludeImports() if includeImportsOverride != nil { includeImports = *includeImportsOverride } - includeWellKnownTypes := currentPluginConfig.IncludeWKT() + includeWellKnownTypes := hashPluginConfig.IncludeWKT() if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } From dc449ef9888f27b5200283cf8246d83c9cefaa18 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 3 Mar 2025 17:17:22 +0100 Subject: [PATCH 22/80] Remove gosec workarounds --- .golangci.yml | 12 ------------ private/bufpkg/bufimage/bufimageutil/image_filter.go | 9 +++++---- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a57119ad1e..2a753eb700 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -357,18 +357,6 @@ issues: # from the fileDescriptor set, the operation should be safe. path: private/bufpkg/bufprotosource/paths.go text: "G115:" - - linters: - - gosec - # bufimageutil is handling images and converting loop indices to int32. Since it is - # parsing from an Image, the operation should be safe. - path: private/bufpkg/bufimage/bufimageutil/bufimageutil.go - text: "G115:" - - linters: - - gosec - # bufimageutil is handling images and converting loop indices to int32. Since it is - # parsing from an Image, the operation should be safe. - path: private/bufpkg/bufimage/bufimageutil/image_filter.go - text: "G115:" - linters: - gosec # Bounds checks have been added with assertion statements to ensure safe int -> int32 diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 01b4d40f43..5c2a1bb067 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -228,15 +228,15 @@ func (b *sourcePathsBuilder) remapFileDescriptor( // Fix the imports to remove any that are no longer used. importsRequired := b.closure.imports[fileDescriptor.GetName()] - indexTo := int32(0) + indexFrom, indexTo := int32(0), int32(0) newFileDescriptor.Dependency = nil dependencyPath := append(sourcePath, fileDependencyTag) dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) - for indexFrom, importPath := range fileDescriptor.Dependency { - path := append(dependencyPath, int32(indexFrom)) + for _, importPath := range fileDescriptor.Dependency { + path := append(dependencyPath, indexFrom) if importsRequired != nil && importsRequired.index(importPath) != -1 { dependencyChanges[indexFrom] = indexTo - if indexTo != int32(indexFrom) { + if indexTo != indexFrom { sourcePathsRemap.markMoved(path, indexTo) } newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, importPath) @@ -247,6 +247,7 @@ func (b *sourcePathsBuilder) remapFileDescriptor( sourcePathsRemap.markDeleted(path) dependencyChanges[indexFrom] = -1 } + indexFrom++ } if importsRequired != nil { newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, importsRequired.keys()...) From 90cc4a076f76e95af95ec5e1efeadbf7e3e99a00 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 3 Mar 2025 17:20:22 +0100 Subject: [PATCH 23/80] Cleanup --- private/bufpkg/bufconfig/generate_plugin_config.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 8fc996065d..6e2b0fc274 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -273,7 +273,7 @@ func newGeneratePluginConfigFromExternalV1Beta1( opt, false, false, - nil, // TODO + nil, strategy, []string{externalConfig.Path}, ) @@ -284,7 +284,7 @@ func newGeneratePluginConfigFromExternalV1Beta1( opt, false, false, - nil, // TODO + nil, strategy, ) } @@ -344,7 +344,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, - nil, // TODO + nil, externalConfig.Revision, ) } @@ -357,7 +357,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, - nil, // TODO + nil, strategy, path, ) @@ -369,7 +369,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, - nil, // TODO + nil, strategy, protocPath, ) @@ -382,7 +382,7 @@ func newGeneratePluginConfigFromExternalV1( opt, false, false, - nil, // TODO + nil, strategy, ) } From cffd97c782de6c007586ee7ab70d8d3bb68f74a3 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 4 Mar 2025 17:26:11 +0100 Subject: [PATCH 24/80] Remove pointers from trailing slice --- private/buf/bufgen/generator.go | 53 +++++++++---------- .../bufimageutil/bufimageutil_test.go | 37 ++++++++++++- .../bufimage/bufimageutil/image_filter.go | 10 +++- 3 files changed, 71 insertions(+), 29 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 0d22d4a38a..f14203633c 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -198,30 +198,6 @@ func (g *generator) generateCode( return nil } -// hashPluginConfigForImage returns a hash of the plugin config for the image. -// This is used to batch plugins that have the same configuration. -// The hash is based on the following properties: -// - ExcludeOptions -// - Strategy -// - RemoteHost -func hashPluginConfigForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { - type imagePluginConfigKey struct { - ExcludeOptions []string - Strategy Strategy - RemoteHost string - } - key := &imagePluginConfigKey{ - ExcludeOptions: pluginConfig.ExcludeOptions(), - Strategy: Strategy(pluginConfig.Strategy()), - RemoteHost: pluginConfig.RemoteHost(), - } - bytes, err := json.Marshal(key) - if err != nil { - return "", err - } - return string(bytes), nil -} - func (g *generator) execPlugins( ctx context.Context, container app.EnvStdioContainer, @@ -235,9 +211,8 @@ func (g *generator) execPlugins( responses := make([]*pluginpb.CodeGeneratorResponse, len(pluginConfigs)) requiredFeatures := computeRequiredFeatures(image) - // Group the pluginConfigs by their hash. The hash only considers the - // properties ExcludeOptions, Strategy, and RemoteHost. - pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, hashPluginConfigForImage) + // Group the pluginConfigs by the properties ExcludeOptions, Strategy, and RemoteHost. + pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, createPluginConfigKeyForImage) if err != nil { return nil, err } @@ -514,3 +489,27 @@ type generateOptions struct { func newGenerateOptions() *generateOptions { return &generateOptions{} } + +// createPluginConfigKeyForImage returns a string of the plugin config with +// a subset of properties. This is used to batch plugins that have similar +// configuration. The key is based on the following properties: +// - ExcludeOptions +// - Strategy +// - RemoteHost +func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { + type pluginConfigKey struct { + ExcludeOptions []string + Strategy Strategy + RemoteHost string + } + key := &pluginConfigKey{ + ExcludeOptions: pluginConfig.ExcludeOptions(), + Strategy: Strategy(pluginConfig.Strategy()), + RemoteHost: pluginConfig.RemoteHost(), + } + bytes, err := json.Marshal(key) + if err != nil { + return "", err + } + return string(bytes), nil +} diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 75503ab50b..3fd1d3045f 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -295,7 +295,7 @@ func TestPackages(t *testing.T) { func TestAny(t *testing.T) { t.Parallel() runDiffTest(t, "testdata/any", "c1.txtar", WithIncludeTypes("ExtendedAnySyntax")) - runDiffTest(t, "testdata/any", "c2.txtar", WithIncludeTypes("ExtendedAnySyntax_InField")) // c2 + runDiffTest(t, "testdata/any", "c2.txtar", WithIncludeTypes("ExtendedAnySyntax_InField")) runDiffTest(t, "testdata/any", "c3.txtar", WithIncludeTypes("ExtendedAnySyntax_InList")) runDiffTest(t, "testdata/any", "c4.txtar", WithIncludeTypes("ExtendedAnySyntax_InMap")) runDiffTest(t, "testdata/any", "d.txtar", WithIncludeTypes("NormalMessageSyntaxValidType")) @@ -388,6 +388,41 @@ func TestTypesFromMainModule(t *testing.T) { assert.ErrorIs(t, err, ErrImageFilterTypeNotFound) } +func TestMutateInPlace(t *testing.T) { + t.Parallel() + ctx := context.Background() + _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/options", bufimage.WithExcludeSourceCodeInfo()) + require.NoError(t, err) + + aProtoFile := image.GetFile("a.proto") + aFileDescriptorProto := aProtoFile.FileDescriptorProto() + assert.Len(t, aFileDescriptorProto.MessageType, 2) // Foo, Empty + assert.Len(t, aFileDescriptorProto.EnumType, 1) // FooEnum + assert.Len(t, aFileDescriptorProto.Service, 1) + + // Shallow copy + shallowFilteredImage, err := FilterImage(image, WithIncludeTypes("pkg.Foo")) + require.NoError(t, err) + + assert.NotSame(t, aFileDescriptorProto, shallowFilteredImage.GetFile("a.proto").FileDescriptorProto()) + + // Mutate in place + mutateFilteredImage, err := FilterImage(image, WithIncludeTypes("pkg.Foo"), WithMutateInPlace()) + require.NoError(t, err) + + // Check that the original image was mutated + assert.Same(t, aFileDescriptorProto, mutateFilteredImage.GetFile("a.proto").FileDescriptorProto()) + assert.Len(t, aFileDescriptorProto.MessageType, 1) // Foo + if assert.GreaterOrEqual(t, cap(aFileDescriptorProto.MessageType), 2) { + slice := aFileDescriptorProto.MessageType[1:cap(aFileDescriptorProto.MessageType)] + for _, elem := range slice { + assert.Nil(t, elem) // Empty + } + } + assert.Nil(t, aFileDescriptorProto.EnumType) + assert.Nil(t, aFileDescriptorProto.Service) +} + func getImage(ctx context.Context, logger *slog.Logger, testdataDir string, options ...bufimage.BuildImageOption) (storage.ReadWriteBucket, bufimage.Image, error) { bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) if err != nil { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 5c2a1bb067..6fd6553501 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -621,7 +621,6 @@ func remapSlice[T any]( if options.mutateInPlace { newList = list[:toIndex:cap(list)] } else { - newList = make([]*T, 0, len(list)) newList = append(newList, list[:toIndex]...) } } @@ -643,6 +642,15 @@ func remapSlice[T any]( sourcePathsRemap.markDeleted(sourcePath) } if isDirty { + if len(newList) == 0 { + return nil, true, nil + } + if options.mutateInPlace && newList != nil { + // Zero out the remaining elements. + for i := int(toIndex); i < len(list); i++ { + list[i] = nil + } + } return newList, true, nil } return list, false, nil From b460387918ff57d3631dd492a404d71a780af13a Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 4 Mar 2025 22:15:18 +0100 Subject: [PATCH 25/80] Drop extension TODO --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 6fd6553501..19f93734c3 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -201,8 +201,6 @@ func (b *sourcePathsBuilder) remapFileDescriptor( return nil, err } isDirty = isDirty || changed - // TODO: extension docs - //newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField) newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField, b.options) if err != nil { return nil, err @@ -257,7 +255,6 @@ func (b *sourcePathsBuilder) remapFileDescriptor( publicDependencyPath := append(sourcePath, filePublicDependencyTag) sourcePathsRemap.markDeleted(publicDependencyPath) - // TODO: validate newFileDescriptor.WeakDependency = nil if len(fileDescriptor.WeakDependency) > 0 { weakDependencyPath := append(sourcePath, fileWeakDependencyTag) @@ -325,8 +322,6 @@ func (b *sourcePathsBuilder) remapDescriptor( newDescriptor.ExtensionRange = newExtensionRange } } - // TODO: sourcePath might not be correct here. - // TODO: extension docs. newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.remapField, b.options) if err != nil { return nil, false, err From e531a7a1794e07b1fea755b54c1403d061b0071d Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 7 Mar 2025 23:32:20 +0100 Subject: [PATCH 26/80] Add types and options for inputs and plugins --- private/buf/bufctl/controller.go | 11 ++- private/buf/bufctl/option.go | 21 ++++ private/buf/bufgen/generator.go | 22 ++++- .../buf/cmd/buf/command/generate/generate.go | 6 ++ private/bufpkg/bufconfig/buf_gen_yaml_file.go | 18 +++- .../bufconfig/generate_plugin_config.go | 96 +++++++++++++++++++ private/bufpkg/bufconfig/input_config.go | 25 +++++ .../bufimage/bufimageutil/bufimageutil.go | 16 +++- 8 files changed, 201 insertions(+), 14 deletions(-) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 32b9e32770..20c0ea0894 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -1331,10 +1331,17 @@ func filterImage( if functionOptions.imageExcludeImports { newImage = bufimage.ImageWithoutImports(newImage) } - if len(functionOptions.imageTypes) > 0 { + includeTypes := functionOptions.imageTypes + excludeTypes := functionOptions.imageExcludeTypes + includeOptions := functionOptions.imageOptions + excludeOptions := functionOptions.imageExcludeOptions + if len(includeTypes) > 0 || len(excludeTypes) > 0 || len(includeOptions) > 0 || len(excludeOptions) > 0 { newImage, err = bufimageutil.FilterImage( newImage, - bufimageutil.WithIncludeTypes(functionOptions.imageTypes...), + bufimageutil.WithIncludeTypes(includeTypes...), + bufimageutil.WithExcludeTypes(excludeTypes...), + bufimageutil.WithIncludeOptions(includeTypes...), + bufimageutil.WithExcludeOptions(excludeOptions...), bufimageutil.WithMutateInPlace(), ) if err != nil { diff --git a/private/buf/bufctl/option.go b/private/buf/bufctl/option.go index 2fd88ec9d7..2697e81ec4 100644 --- a/private/buf/bufctl/option.go +++ b/private/buf/bufctl/option.go @@ -72,6 +72,24 @@ func WithImageTypes(imageTypes []string) FunctionOption { } } +func WithImageExcludeTypes(imageExcludeTypes []string) FunctionOption { + return func(functionOptions *functionOptions) { + functionOptions.imageExcludeTypes = imageExcludeTypes + } +} + +func WithImageOptions(imageOptions []string) FunctionOption { + return func(functionOptions *functionOptions) { + functionOptions.imageOptions = imageOptions + } +} + +func WithImageExcludeOptions(imageExcludeOptions []string) FunctionOption { + return func(functionOptions *functionOptions) { + functionOptions.imageExcludeOptions = imageExcludeOptions + } +} + func WithImageAsFileDescriptorSet(imageAsFileDescriptorSet bool) FunctionOption { return func(functionOptions *functionOptions) { functionOptions.imageAsFileDescriptorSet = imageAsFileDescriptorSet @@ -134,6 +152,9 @@ type functionOptions struct { imageExcludeSourceInfo bool imageExcludeImports bool imageTypes []string + imageExcludeTypes []string + imageOptions []string + imageExcludeOptions []string imageAsFileDescriptorSet bool configOverride string ignoreAndDisallowV1BufWorkYAMLs bool diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index f14203633c..00bef7e774 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -212,6 +212,7 @@ func (g *generator) execPlugins( requiredFeatures := computeRequiredFeatures(image) // Group the pluginConfigs by the properties ExcludeOptions, Strategy, and RemoteHost. + pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, createPluginConfigKeyForImage) if err != nil { return nil, err @@ -221,10 +222,17 @@ func (g *generator) execPlugins( hashPluginConfig := indexedPluginConfigs[0].Value // Apply per-config filters. - if excludeOptions := hashPluginConfig.ExcludeOptions(); len(excludeOptions) > 0 { + includeTypes := hashPluginConfig.Types() + excludeTypes := hashPluginConfig.ExcludeTypes() + includeOptions := hashPluginConfig.Options() + excludeOptions := hashPluginConfig.ExcludeOptions() + if len(includeTypes) > 0 || len(excludeTypes) > 0 || len(includeOptions) > 0 || len(excludeOptions) > 0 { var err error image, err = bufimageutil.FilterImage( image, + bufimageutil.WithIncludeTypes(includeTypes...), + bufimageutil.WithExcludeTypes(excludeTypes...), + bufimageutil.WithIncludeOptions(includeOptions...), bufimageutil.WithExcludeOptions(excludeOptions...), ) if err != nil { @@ -498,11 +506,17 @@ func newGenerateOptions() *generateOptions { // - RemoteHost func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { type pluginConfigKey struct { - ExcludeOptions []string - Strategy Strategy - RemoteHost string + Types []string `json:"types"` + ExcludeTypes []string `json:"exclude_types"` + Options []string `json:"options"` + ExcludeOptions []string `json:"exclude_options"` + Strategy Strategy `json:"strategy"` + RemoteHost string `json:"remote_host"` } key := &pluginConfigKey{ + Types: pluginConfig.Types(), + ExcludeTypes: pluginConfig.ExcludeTypes(), + Options: pluginConfig.Options(), ExcludeOptions: pluginConfig.ExcludeOptions(), Strategy: Strategy(pluginConfig.Strategy()), RemoteHost: pluginConfig.RemoteHost(), diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index ef20638987..446ee7f208 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -634,12 +634,18 @@ func getInputImages( if len(includeTypesOverride) > 0 { includeTypes = includeTypesOverride } + excludeTypes := inputConfig.ExcludeTypes() + includeOptions := inputConfig.IncludeOptions() + excludeOptions := inputConfig.ExcludeOptions() inputImage, err := controller.GetImageForInputConfig( ctx, inputConfig, bufctl.WithConfigOverride(moduleConfigOverride), bufctl.WithTargetPaths(targetPaths, excludePaths), bufctl.WithImageTypes(includeTypes), + bufctl.WithImageExcludeTypes(excludeTypes), + bufctl.WithImageOptions(includeOptions), + bufctl.WithImageExcludeOptions(excludeOptions), ) if err != nil { return nil, err diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 737bcee0ba..a4369858de 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -514,6 +514,13 @@ type externalGeneratePluginConfigV2 struct { IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` // Strategy is only valid with ProtoBuiltin and Local. Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` + + // Types is a list of types to include in the image. + Types []string `json:"types,omitempty" yaml:"types,omitempty"` + // ExcludeTypes removes types from the image. + ExcludeTypes []string `json:"exclude_types,omitempty" yaml:"exclude_types,omitempty"` + // Options is a list of options to include in the image. + Options []string `json:"options,omitempty" yaml:"options,omitempty"` // ExcludeOptions removes options from the image. ExcludeOptions []string `json:"exclude_options,omitempty" yaml:"exclude_options,omitempty"` } @@ -566,10 +573,13 @@ type externalInputConfigV2 struct { TextImage *string `json:"text_image,omitempty" yaml:"text_image,omitempty"` YAMLImage *string `json:"yaml_image,omitempty" yaml:"yaml_image,omitempty"` GitRepo *string `json:"git_repo,omitempty" yaml:"git_repo,omitempty"` - // Types, TargetPaths and ExcludePaths are available for all formats. - Types []string `json:"types,omitempty" yaml:"types,omitempty"` - TargetPaths []string `json:"paths,omitempty" yaml:"paths,omitempty"` - ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` + // Types, ExcludeTypes, Options, ExcludeOptions, TargetPaths and ExcludePaths are available for all formats. + Types []string `json:"types,omitempty" yaml:"types,omitempty"` + ExcludeTypes []string `json:"exclude_types,omitempty" yaml:"exclude_types,omitempty"` + Options []string `json:"options,omitempty" yaml:"options,omitempty"` + ExcludeOptions []string `json:"exclude_options,omitempty" yaml:"exclude_options,omitempty"` + TargetPaths []string `json:"paths,omitempty" yaml:"paths,omitempty"` + ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` // The following options are available depending on input format. Compression *string `json:"compression,omitempty" yaml:"compression,omitempty"` StripComponents *uint32 `json:"strip_components,omitempty" yaml:"strip_components,omitempty"` diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 6e2b0fc274..add2f4ca85 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -112,6 +112,12 @@ type GeneratePluginConfig interface { // // This is not empty only when the plugin is remote. Revision() int + // Types returns the types to include. + Types() []string + // ExcludeTypes returns the types to exclude. + ExcludeTypes() []string + // Options returns the options to include. + Options() []string // ExcludeOptions returns the options to exclude. ExcludeOptions() []string @@ -125,6 +131,9 @@ func NewRemoteGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + types []string, + excludeTypes []string, + options []string, excludeOptions []string, revision int, ) (GeneratePluginConfig, error) { @@ -134,6 +143,9 @@ func NewRemoteGeneratePluginConfig( opt, includeImports, includeWKT, + types, + excludeTypes, + options, excludeOptions, revision, ) @@ -146,6 +158,9 @@ func NewLocalOrProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + includeTypes []string, + excludeTypes []string, + includeOptions []string, excludeOptions []string, strategy *GenerateStrategy, ) (GeneratePluginConfig, error) { @@ -155,6 +170,9 @@ func NewLocalOrProtocBuiltinGeneratePluginConfig( opt, includeImports, includeWKT, + includeTypes, + excludeTypes, + includeOptions, excludeOptions, strategy, ) @@ -167,6 +185,9 @@ func NewLocalGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + types []string, + excludeTypes []string, + options []string, excludeOptions []string, strategy *GenerateStrategy, path []string, @@ -177,6 +198,9 @@ func NewLocalGeneratePluginConfig( opt, includeImports, includeWKT, + types, + excludeTypes, + options, excludeOptions, strategy, path, @@ -191,6 +215,9 @@ func NewProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + includeTypes []string, + excludeTypes []string, + includeOptions []string, excludeOptions []string, strategy *GenerateStrategy, protocPath []string, @@ -201,6 +228,9 @@ func NewProtocBuiltinGeneratePluginConfig( opt, includeImports, includeWKT, + includeTypes, + excludeTypes, + includeOptions, excludeOptions, strategy, protocPath, @@ -241,6 +271,9 @@ type generatePluginConfig struct { opts []string includeImports bool includeWKT bool + types []string + excludeTypes []string + options []string excludeOptions []string strategy *GenerateStrategy path []string @@ -274,6 +307,9 @@ func newGeneratePluginConfigFromExternalV1Beta1( false, false, nil, + nil, + nil, + nil, strategy, []string{externalConfig.Path}, ) @@ -285,6 +321,9 @@ func newGeneratePluginConfigFromExternalV1Beta1( false, false, nil, + nil, + nil, + nil, strategy, ) } @@ -345,6 +384,9 @@ func newGeneratePluginConfigFromExternalV1( false, false, nil, + nil, + nil, + nil, externalConfig.Revision, ) } @@ -358,6 +400,9 @@ func newGeneratePluginConfigFromExternalV1( false, false, nil, + nil, + nil, + nil, strategy, path, ) @@ -370,6 +415,9 @@ func newGeneratePluginConfigFromExternalV1( false, false, nil, + nil, + nil, + nil, strategy, protocPath, ) @@ -383,6 +431,9 @@ func newGeneratePluginConfigFromExternalV1( false, false, nil, + nil, + nil, + nil, strategy, ) } @@ -439,6 +490,9 @@ func newGeneratePluginConfigFromExternalV2( opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + externalConfig.Types, + externalConfig.ExcludeTypes, + externalConfig.Options, externalConfig.ExcludeOptions, revision, ) @@ -460,6 +514,9 @@ func newGeneratePluginConfigFromExternalV2( opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + externalConfig.Types, + externalConfig.ExcludeTypes, + externalConfig.Options, externalConfig.ExcludeOptions, parsedStrategy, path, @@ -478,6 +535,9 @@ func newGeneratePluginConfigFromExternalV2( opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + externalConfig.Types, + externalConfig.ExcludeTypes, + externalConfig.Options, externalConfig.ExcludeOptions, parsedStrategy, protocPath, @@ -493,6 +553,9 @@ func newRemoteGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + types []string, + excludeTypes []string, + options []string, excludeOptions []string, revision int, ) (*generatePluginConfig, error) { @@ -515,6 +578,9 @@ func newRemoteGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, + types: types, + excludeTypes: excludeTypes, + options: options, excludeOptions: excludeOptions, }, nil } @@ -525,6 +591,9 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + types []string, + excludeTypes []string, + options []string, excludeOptions []string, strategy *GenerateStrategy, ) (*generatePluginConfig, error) { @@ -539,6 +608,9 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, + types: types, + excludeTypes: excludeTypes, + options: options, excludeOptions: excludeOptions, }, nil } @@ -549,6 +621,9 @@ func newLocalGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + types []string, + excludeTypes []string, + options []string, excludeOptions []string, strategy *GenerateStrategy, path []string, @@ -568,6 +643,9 @@ func newLocalGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, + types: types, + excludeTypes: excludeTypes, + options: options, excludeOptions: excludeOptions, }, nil } @@ -578,6 +656,9 @@ func newProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, + types []string, + excludeTypes []string, + options []string, excludeOptions []string, strategy *GenerateStrategy, protocPath []string, @@ -594,6 +675,9 @@ func newProtocBuiltinGeneratePluginConfig( strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, + types: types, + excludeTypes: excludeTypes, + options: options, excludeOptions: excludeOptions, }, nil } @@ -622,6 +706,18 @@ func (p *generatePluginConfig) IncludeWKT() bool { return p.includeWKT } +func (p *generatePluginConfig) Types() []string { + return p.types +} + +func (p *generatePluginConfig) ExcludeTypes() []string { + return p.excludeTypes +} + +func (p *generatePluginConfig) Options() []string { + return p.options +} + func (p *generatePluginConfig) ExcludeOptions() []string { return p.excludeOptions } diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index e4cdc95f0c..7ad2874895 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -161,7 +161,17 @@ type InputConfig interface { // ExcludePaths returns paths not to generate for. ExcludePaths() []string // IncludeTypes returns the types to generate. An empty slice means to generate for all types. + // This is unioned with ExcludeTypes. IncludeTypes() []string + // ExcludeTypes returns the types to exclude. An empty slice means to exclude no types. + // This is unioned with IncludeTypes. + ExcludeTypes() []string + // IncludeOptions returns the options to include. An empty slice means to include all options. + // This is unioned with ExcludeOptions. + IncludeOptions() []string + // ExcludeOptions returns the options to exclude. An empty slice means to exclude no options. + // This is unioned with IncludeOptions. + ExcludeOptions() []string isInputConfig() } @@ -343,6 +353,9 @@ type inputConfig struct { recurseSubmodules bool includePackageFiles bool includeTypes []string + excludeTypes []string + includeOptions []string + excludeOptions []string targetPaths []string excludePaths []string } @@ -516,6 +529,18 @@ func (i *inputConfig) IncludeTypes() []string { return i.includeTypes } +func (i *inputConfig) ExcludeTypes() []string { + return i.excludeTypes +} + +func (i *inputConfig) IncludeOptions() []string { + return i.includeOptions +} + +func (i *inputConfig) ExcludeOptions() []string { + return i.excludeOptions +} + func (i *inputConfig) isInputConfig() {} func newExternalInputConfigV2FromInputConfig( diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 5c7a9b5223..a7a0f24f01 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -108,7 +108,7 @@ func WithAllowFilterByImportedType() ImageFilterOption { // [ErrImageFilterTypeNotFound] will be returned. func WithIncludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { - if opts.includeTypes == nil { + if len(typeNames) > 0 && opts.includeTypes == nil { opts.includeTypes = make(map[string]struct{}, len(typeNames)) } for _, typeName := range typeNames { @@ -126,7 +126,7 @@ func WithIncludeTypes(typeNames ...string) ImageFilterOption { // [ErrImageFilterTypeNotFound] will be returned. func WithExcludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { - if opts.excludeTypes == nil { + if len(typeNames) > 0 && opts.excludeTypes == nil { opts.excludeTypes = make(map[string]struct{}, len(typeNames)) } for _, typeName := range typeNames { @@ -143,7 +143,7 @@ func WithExcludeTypes(typeNames ...string) ImageFilterOption { // If the option does not exist in the image, it will be ignored. func WithIncludeOptions(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { - if opts.includeOptions == nil { + if len(typeNames) > 0 && opts.includeOptions == nil { opts.includeOptions = make(map[string]struct{}, len(typeNames)) } for _, typeName := range typeNames { @@ -160,7 +160,7 @@ func WithIncludeOptions(typeNames ...string) ImageFilterOption { // If the option does not exist in the image, it will be ignored. func WithExcludeOptions(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { - if opts.excludeOptions == nil { + if len(typeNames) > 0 && opts.excludeOptions == nil { opts.excludeOptions = make(map[string]struct{}, len(typeNames)) } for _, typeName := range typeNames { @@ -265,6 +265,14 @@ func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.I for _, option := range options { option(filterOptions) } + if filterOptions.includeCustomOptions && + filterOptions.includeKnownExtensions && + len(filterOptions.excludeTypes) == 0 && + len(filterOptions.includeTypes) == 0 && + len(filterOptions.excludeOptions) == 0 && + len(filterOptions.includeOptions) == 0 { + return image, nil + } return filterImage(image, filterOptions) } From 2a2017a76f385d8c11fbb97f6631f66dcfbd9204 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 10 Mar 2025 14:49:08 +0000 Subject: [PATCH 27/80] Sort key parts --- private/buf/bufgen/generator.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 00bef7e774..d87a11e446 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "path/filepath" + "sort" connect "connectrpc.com/connect" "github.com/bufbuild/buf/private/buf/bufprotopluginexec" @@ -521,6 +522,10 @@ func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) Strategy: Strategy(pluginConfig.Strategy()), RemoteHost: pluginConfig.RemoteHost(), } + sort.Strings(key.Types) + sort.Strings(key.ExcludeTypes) + sort.Strings(key.Options) + sort.Strings(key.ExcludeOptions) bytes, err := json.Marshal(key) if err != nil { return "", err From 49cf91ce603c2e32ae13d0e8832067bb64cc0687 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 09:55:15 +0000 Subject: [PATCH 28/80] Only exclusion of types --- private/buf/bufctl/controller.go | 2 - private/buf/bufctl/option.go | 12 - private/buf/bufgen/generator.go | 28 +- .../buf/cmd/buf/command/generate/generate.go | 4 - private/bufpkg/bufconfig/buf_gen_yaml_file.go | 16 +- .../bufconfig/buf_gen_yaml_file_test.go | 8 +- .../bufconfig/generate_plugin_config.go | 68 --- private/bufpkg/bufconfig/input_config.go | 16 - .../bufimage/bufimageutil/bufimageutil.go | 93 ++- .../bufimageutil/bufimageutil_test.go | 47 +- .../bufimage/bufimageutil/image_filter.go | 2 +- .../bufimageutil/testdata/imports/bar.txtar | 400 +++---------- .../bufimageutil/testdata/imports/foo.txtar | 1 - .../testdata/imports/none_excluded.txtar | 486 --------------- .../options/options.foo.exclude.txtar | 8 - .../options/options.foo.include.txtar | 463 --------------- .../options/options.jstype.exclude.txtar | 558 ------------------ .../options/options.jstype.include.txtar | 453 -------------- .../options/options.message.include.txtar | 456 -------------- .../testdata/options/options.only_file.txtar | 35 -- 20 files changed, 148 insertions(+), 3008 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 20c0ea0894..f2f827366c 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -1340,8 +1340,6 @@ func filterImage( newImage, bufimageutil.WithIncludeTypes(includeTypes...), bufimageutil.WithExcludeTypes(excludeTypes...), - bufimageutil.WithIncludeOptions(includeTypes...), - bufimageutil.WithExcludeOptions(excludeOptions...), bufimageutil.WithMutateInPlace(), ) if err != nil { diff --git a/private/buf/bufctl/option.go b/private/buf/bufctl/option.go index 2697e81ec4..a0f07ddbd8 100644 --- a/private/buf/bufctl/option.go +++ b/private/buf/bufctl/option.go @@ -78,18 +78,6 @@ func WithImageExcludeTypes(imageExcludeTypes []string) FunctionOption { } } -func WithImageOptions(imageOptions []string) FunctionOption { - return func(functionOptions *functionOptions) { - functionOptions.imageOptions = imageOptions - } -} - -func WithImageExcludeOptions(imageExcludeOptions []string) FunctionOption { - return func(functionOptions *functionOptions) { - functionOptions.imageExcludeOptions = imageExcludeOptions - } -} - func WithImageAsFileDescriptorSet(imageAsFileDescriptorSet bool) FunctionOption { return func(functionOptions *functionOptions) { functionOptions.imageAsFileDescriptorSet = imageAsFileDescriptorSet diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index d87a11e446..589c4ed610 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -225,16 +225,12 @@ func (g *generator) execPlugins( // Apply per-config filters. includeTypes := hashPluginConfig.Types() excludeTypes := hashPluginConfig.ExcludeTypes() - includeOptions := hashPluginConfig.Options() - excludeOptions := hashPluginConfig.ExcludeOptions() - if len(includeTypes) > 0 || len(excludeTypes) > 0 || len(includeOptions) > 0 || len(excludeOptions) > 0 { + if len(includeTypes) > 0 || len(excludeTypes) > 0 { var err error image, err = bufimageutil.FilterImage( image, bufimageutil.WithIncludeTypes(includeTypes...), bufimageutil.WithExcludeTypes(excludeTypes...), - bufimageutil.WithIncludeOptions(includeOptions...), - bufimageutil.WithExcludeOptions(excludeOptions...), ) if err != nil { return nil, err @@ -507,25 +503,19 @@ func newGenerateOptions() *generateOptions { // - RemoteHost func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { type pluginConfigKey struct { - Types []string `json:"types"` - ExcludeTypes []string `json:"exclude_types"` - Options []string `json:"options"` - ExcludeOptions []string `json:"exclude_options"` - Strategy Strategy `json:"strategy"` - RemoteHost string `json:"remote_host"` + Types []string `json:"types"` + ExcludeTypes []string `json:"exclude_types"` + Strategy Strategy `json:"strategy"` + RemoteHost string `json:"remote_host"` } key := &pluginConfigKey{ - Types: pluginConfig.Types(), - ExcludeTypes: pluginConfig.ExcludeTypes(), - Options: pluginConfig.Options(), - ExcludeOptions: pluginConfig.ExcludeOptions(), - Strategy: Strategy(pluginConfig.Strategy()), - RemoteHost: pluginConfig.RemoteHost(), + Types: pluginConfig.Types(), + ExcludeTypes: pluginConfig.ExcludeTypes(), + Strategy: Strategy(pluginConfig.Strategy()), + RemoteHost: pluginConfig.RemoteHost(), } sort.Strings(key.Types) sort.Strings(key.ExcludeTypes) - sort.Strings(key.Options) - sort.Strings(key.ExcludeOptions) bytes, err := json.Marshal(key) if err != nil { return "", err diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 446ee7f208..d2552f9ec0 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -635,8 +635,6 @@ func getInputImages( includeTypes = includeTypesOverride } excludeTypes := inputConfig.ExcludeTypes() - includeOptions := inputConfig.IncludeOptions() - excludeOptions := inputConfig.ExcludeOptions() inputImage, err := controller.GetImageForInputConfig( ctx, inputConfig, @@ -644,8 +642,6 @@ func getInputImages( bufctl.WithTargetPaths(targetPaths, excludePaths), bufctl.WithImageTypes(includeTypes), bufctl.WithImageExcludeTypes(excludeTypes), - bufctl.WithImageOptions(includeOptions), - bufctl.WithImageExcludeOptions(excludeOptions), ) if err != nil { return nil, err diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index a4369858de..e2c40a7bcb 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -519,10 +519,6 @@ type externalGeneratePluginConfigV2 struct { Types []string `json:"types,omitempty" yaml:"types,omitempty"` // ExcludeTypes removes types from the image. ExcludeTypes []string `json:"exclude_types,omitempty" yaml:"exclude_types,omitempty"` - // Options is a list of options to include in the image. - Options []string `json:"options,omitempty" yaml:"options,omitempty"` - // ExcludeOptions removes options from the image. - ExcludeOptions []string `json:"exclude_options,omitempty" yaml:"exclude_options,omitempty"` } // externalGenerateManagedConfigV2 represents the managed mode config in a v2 buf.gen.yaml file. @@ -573,13 +569,11 @@ type externalInputConfigV2 struct { TextImage *string `json:"text_image,omitempty" yaml:"text_image,omitempty"` YAMLImage *string `json:"yaml_image,omitempty" yaml:"yaml_image,omitempty"` GitRepo *string `json:"git_repo,omitempty" yaml:"git_repo,omitempty"` - // Types, ExcludeTypes, Options, ExcludeOptions, TargetPaths and ExcludePaths are available for all formats. - Types []string `json:"types,omitempty" yaml:"types,omitempty"` - ExcludeTypes []string `json:"exclude_types,omitempty" yaml:"exclude_types,omitempty"` - Options []string `json:"options,omitempty" yaml:"options,omitempty"` - ExcludeOptions []string `json:"exclude_options,omitempty" yaml:"exclude_options,omitempty"` - TargetPaths []string `json:"paths,omitempty" yaml:"paths,omitempty"` - ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` + // Types, ExcludeTypes, TargetPaths and ExcludePaths are available for all formats. + Types []string `json:"types,omitempty" yaml:"types,omitempty"` + ExcludeTypes []string `json:"exclude_types,omitempty" yaml:"exclude_types,omitempty"` + TargetPaths []string `json:"paths,omitempty" yaml:"paths,omitempty"` + ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` // The following options are available depending on input format. Compression *string `json:"compression,omitempty" yaml:"compression,omitempty"` StripComponents *uint32 `json:"strip_components,omitempty" yaml:"strip_components,omitempty"` diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go b/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go index 4903120688..e178233a42 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file_test.go @@ -180,8 +180,12 @@ plugins: strategy: all include_imports: true include_wkt: true - exclude_options: + types: + - "foo.v1.User" + exclude_types: - buf.validate.oneof + - buf.validate.message + - buf.validate.field inputs: - git_repo: github.com/acme/weather branch: dev @@ -191,6 +195,8 @@ inputs: types: - "foo.v1.User" - "foo.v1.UserService" + exclude_types: + - buf.validate.oneof paths: - a/b/c - a/b/d diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index add2f4ca85..f6c39a947c 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -116,10 +116,6 @@ type GeneratePluginConfig interface { Types() []string // ExcludeTypes returns the types to exclude. ExcludeTypes() []string - // Options returns the options to include. - Options() []string - // ExcludeOptions returns the options to exclude. - ExcludeOptions() []string isGeneratePluginConfig() } @@ -133,8 +129,6 @@ func NewRemoteGeneratePluginConfig( includeWKT bool, types []string, excludeTypes []string, - options []string, - excludeOptions []string, revision int, ) (GeneratePluginConfig, error) { return newRemoteGeneratePluginConfig( @@ -145,8 +139,6 @@ func NewRemoteGeneratePluginConfig( includeWKT, types, excludeTypes, - options, - excludeOptions, revision, ) } @@ -160,8 +152,6 @@ func NewLocalOrProtocBuiltinGeneratePluginConfig( includeWKT bool, includeTypes []string, excludeTypes []string, - includeOptions []string, - excludeOptions []string, strategy *GenerateStrategy, ) (GeneratePluginConfig, error) { return newLocalOrProtocBuiltinGeneratePluginConfig( @@ -172,8 +162,6 @@ func NewLocalOrProtocBuiltinGeneratePluginConfig( includeWKT, includeTypes, excludeTypes, - includeOptions, - excludeOptions, strategy, ) } @@ -187,8 +175,6 @@ func NewLocalGeneratePluginConfig( includeWKT bool, types []string, excludeTypes []string, - options []string, - excludeOptions []string, strategy *GenerateStrategy, path []string, ) (GeneratePluginConfig, error) { @@ -200,8 +186,6 @@ func NewLocalGeneratePluginConfig( includeWKT, types, excludeTypes, - options, - excludeOptions, strategy, path, ) @@ -217,8 +201,6 @@ func NewProtocBuiltinGeneratePluginConfig( includeWKT bool, includeTypes []string, excludeTypes []string, - includeOptions []string, - excludeOptions []string, strategy *GenerateStrategy, protocPath []string, ) (GeneratePluginConfig, error) { @@ -230,8 +212,6 @@ func NewProtocBuiltinGeneratePluginConfig( includeWKT, includeTypes, excludeTypes, - includeOptions, - excludeOptions, strategy, protocPath, ) @@ -243,7 +223,6 @@ func NewGeneratePluginConfigWithIncludeImportsAndWKT( config GeneratePluginConfig, includeImports bool, includeWKT bool, - excludeOptions []string, ) (GeneratePluginConfig, error) { originalConfig, ok := config.(*generatePluginConfig) if !ok { @@ -256,9 +235,6 @@ func NewGeneratePluginConfigWithIncludeImportsAndWKT( if includeWKT { generatePluginConfig.includeWKT = true } - if len(excludeOptions) > 0 { - generatePluginConfig.excludeOptions = excludeOptions - } return &generatePluginConfig, nil } @@ -273,8 +249,6 @@ type generatePluginConfig struct { includeWKT bool types []string excludeTypes []string - options []string - excludeOptions []string strategy *GenerateStrategy path []string protocPath []string @@ -308,8 +282,6 @@ func newGeneratePluginConfigFromExternalV1Beta1( false, nil, nil, - nil, - nil, strategy, []string{externalConfig.Path}, ) @@ -322,8 +294,6 @@ func newGeneratePluginConfigFromExternalV1Beta1( false, nil, nil, - nil, - nil, strategy, ) } @@ -385,8 +355,6 @@ func newGeneratePluginConfigFromExternalV1( false, nil, nil, - nil, - nil, externalConfig.Revision, ) } @@ -401,8 +369,6 @@ func newGeneratePluginConfigFromExternalV1( false, nil, nil, - nil, - nil, strategy, path, ) @@ -416,8 +382,6 @@ func newGeneratePluginConfigFromExternalV1( false, nil, nil, - nil, - nil, strategy, protocPath, ) @@ -432,8 +396,6 @@ func newGeneratePluginConfigFromExternalV1( false, nil, nil, - nil, - nil, strategy, ) } @@ -492,8 +454,6 @@ func newGeneratePluginConfigFromExternalV2( externalConfig.IncludeWKT, externalConfig.Types, externalConfig.ExcludeTypes, - externalConfig.Options, - externalConfig.ExcludeOptions, revision, ) case externalConfig.Local != nil: @@ -516,8 +476,6 @@ func newGeneratePluginConfigFromExternalV2( externalConfig.IncludeWKT, externalConfig.Types, externalConfig.ExcludeTypes, - externalConfig.Options, - externalConfig.ExcludeOptions, parsedStrategy, path, ) @@ -537,8 +495,6 @@ func newGeneratePluginConfigFromExternalV2( externalConfig.IncludeWKT, externalConfig.Types, externalConfig.ExcludeTypes, - externalConfig.Options, - externalConfig.ExcludeOptions, parsedStrategy, protocPath, ) @@ -555,8 +511,6 @@ func newRemoteGeneratePluginConfig( includeWKT bool, types []string, excludeTypes []string, - options []string, - excludeOptions []string, revision int, ) (*generatePluginConfig, error) { if includeWKT && !includeImports { @@ -580,8 +534,6 @@ func newRemoteGeneratePluginConfig( includeWKT: includeWKT, types: types, excludeTypes: excludeTypes, - options: options, - excludeOptions: excludeOptions, }, nil } @@ -593,8 +545,6 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( includeWKT bool, types []string, excludeTypes []string, - options []string, - excludeOptions []string, strategy *GenerateStrategy, ) (*generatePluginConfig, error) { if includeWKT && !includeImports { @@ -610,8 +560,6 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( includeWKT: includeWKT, types: types, excludeTypes: excludeTypes, - options: options, - excludeOptions: excludeOptions, }, nil } @@ -623,8 +571,6 @@ func newLocalGeneratePluginConfig( includeWKT bool, types []string, excludeTypes []string, - options []string, - excludeOptions []string, strategy *GenerateStrategy, path []string, ) (*generatePluginConfig, error) { @@ -645,8 +591,6 @@ func newLocalGeneratePluginConfig( includeWKT: includeWKT, types: types, excludeTypes: excludeTypes, - options: options, - excludeOptions: excludeOptions, }, nil } @@ -658,8 +602,6 @@ func newProtocBuiltinGeneratePluginConfig( includeWKT bool, types []string, excludeTypes []string, - options []string, - excludeOptions []string, strategy *GenerateStrategy, protocPath []string, ) (*generatePluginConfig, error) { @@ -677,8 +619,6 @@ func newProtocBuiltinGeneratePluginConfig( includeWKT: includeWKT, types: types, excludeTypes: excludeTypes, - options: options, - excludeOptions: excludeOptions, }, nil } @@ -714,14 +654,6 @@ func (p *generatePluginConfig) ExcludeTypes() []string { return p.excludeTypes } -func (p *generatePluginConfig) Options() []string { - return p.options -} - -func (p *generatePluginConfig) ExcludeOptions() []string { - return p.excludeOptions -} - func (p *generatePluginConfig) Strategy() GenerateStrategy { if p.strategy == nil { return GenerateStrategyDirectory diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 7ad2874895..aeb7a57079 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -166,12 +166,6 @@ type InputConfig interface { // ExcludeTypes returns the types to exclude. An empty slice means to exclude no types. // This is unioned with IncludeTypes. ExcludeTypes() []string - // IncludeOptions returns the options to include. An empty slice means to include all options. - // This is unioned with ExcludeOptions. - IncludeOptions() []string - // ExcludeOptions returns the options to exclude. An empty slice means to exclude no options. - // This is unioned with IncludeOptions. - ExcludeOptions() []string isInputConfig() } @@ -354,8 +348,6 @@ type inputConfig struct { includePackageFiles bool includeTypes []string excludeTypes []string - includeOptions []string - excludeOptions []string targetPaths []string excludePaths []string } @@ -533,14 +525,6 @@ func (i *inputConfig) ExcludeTypes() []string { return i.excludeTypes } -func (i *inputConfig) IncludeOptions() []string { - return i.includeOptions -} - -func (i *inputConfig) ExcludeOptions() []string { - return i.excludeOptions -} - func (i *inputConfig) isInputConfig() {} func newExternalInputConfigV2FromInputConfig( diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index a7a0f24f01..e9ac26b853 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -135,40 +135,6 @@ func WithExcludeTypes(typeNames ...string) ImageFilterOption { } } -// WithIncludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies -// the set of options that should be included in the filtered image. -// -// May be provided multiple times. The option names should be fully qualified. -// For example, "google.protobuf.FieldOptions.jstype" or "buf.validate.field". -// If the option does not exist in the image, it will be ignored. -func WithIncludeOptions(typeNames ...string) ImageFilterOption { - return func(opts *imageFilterOptions) { - if len(typeNames) > 0 && opts.includeOptions == nil { - opts.includeOptions = make(map[string]struct{}, len(typeNames)) - } - for _, typeName := range typeNames { - opts.includeOptions[typeName] = struct{}{} - } - } -} - -// WithExcludeOptions returns an option for ImageFilteredByTypesWithOptions that specifies -// the set of options that should be excluded from the filtered image. -// -// May be provided multiple times. The option names should be fully qualified. -// For example, "google.protobuf.FieldOptions.jstype" or "buf.validate.field". -// If the option does not exist in the image, it will be ignored. -func WithExcludeOptions(typeNames ...string) ImageFilterOption { - return func(opts *imageFilterOptions) { - if len(typeNames) > 0 && opts.excludeOptions == nil { - opts.excludeOptions = make(map[string]struct{}, len(typeNames)) - } - for _, typeName := range typeNames { - opts.excludeOptions[typeName] = struct{}{} - } - } -} - // WithMutateInPlace returns an option for ImageFilteredByTypesWithOptions that specifies // that the filtered image should be mutated in place. This option is useful when the // unfiltered image is no longer needed and the caller wants to avoid the overhead of @@ -265,12 +231,11 @@ func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.I for _, option := range options { option(filterOptions) } + // Check for defaults that would result in no filtering. if filterOptions.includeCustomOptions && filterOptions.includeKnownExtensions && len(filterOptions.excludeTypes) == 0 && - len(filterOptions.includeTypes) == 0 && - len(filterOptions.excludeOptions) == 0 && - len(filterOptions.includeOptions) == 0 { + len(filterOptions.includeTypes) == 0 { return image, nil } return filterImage(image, filterOptions) @@ -352,25 +317,28 @@ func (t *transitiveClosure) hasType( func (t *transitiveClosure) hasOption( fieldDescriptor protoreflect.FieldDescriptor, + imageIndex *imageIndex, options *imageFilterOptions, ) (isIncluded bool) { - fullName := fieldDescriptor.FullName() if options == nil { return true // no filter } - if options.excludeOptions != nil { - if _, ok := options.excludeOptions[string(fullName)]; ok { - return false - } + if !fieldDescriptor.IsExtension() { + return true } - if options.includeOptions != nil { - _, isIncluded = options.includeOptions[string(fullName)] - return isIncluded + if !options.includeCustomOptions { + return false } - if fieldDescriptor.IsExtension() && !options.includeCustomOptions { + fullName := fieldDescriptor.FullName() + descriptor := imageIndex.ByName[fullName].element + switch mode := t.elements[descriptor]; mode { + case inclusionModeExplicit, inclusionModeImplicit, inclusionModeEnclosing, inclusionModeUnknown: + return true + case inclusionModeExcluded: + return false + default: return false } - return true } func (t *transitiveClosure) includeType( @@ -382,9 +350,13 @@ func (t *transitiveClosure) includeType( // matching, such as ability to get a package AND all of its sub-packages. descriptorInfo, ok := imageIndex.ByName[typeName] if ok { + // If an extension field, error. Extension fields can not be included only excluded. + if field, ok := descriptorInfo.element.(*descriptorpb.FieldDescriptorProto); ok && field.GetExtendee() != "" { + return fmt.Errorf("inclusion of type %q: extension fields can not be included", typeName) + } // It's a type name if !options.allowImportedTypes && descriptorInfo.file.IsImport() { - return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + return fmt.Errorf("inclusion of type %q: %w", typeName, ErrImageFilterTypeIsImport) } return t.addElement(descriptorInfo.element, "", false, imageIndex, options) } @@ -392,7 +364,7 @@ func (t *transitiveClosure) includeType( pkg, ok := imageIndex.Packages[string(typeName)] if !ok { // but it's not... - return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeNotFound) + return fmt.Errorf("inclusion of type %q: %w", typeName, ErrImageFilterTypeNotFound) } if !options.allowImportedTypes { // if package includes only imported files, then reject @@ -404,7 +376,7 @@ func (t *transitiveClosure) includeType( } } if onlyImported { - return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + return fmt.Errorf("inclusion of type %q: %w", typeName, ErrImageFilterTypeIsImport) } } for _, file := range pkg.files { @@ -581,17 +553,13 @@ func (t *transitiveClosure) excludeType( ) error { descriptorInfo, ok := imageIndex.ByName[typeName] if ok { - // It's a type name - if !options.allowImportedTypes && descriptorInfo.file.IsImport() { - return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) - } return t.excludeElement(descriptorInfo.element, imageIndex, options) } // It could be a package name pkg, ok := imageIndex.Packages[string(typeName)] if !ok { // but it's not... - return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeNotFound) + return fmt.Errorf("exclusion of type %q: %w", typeName, ErrImageFilterTypeNotFound) } if !options.allowImportedTypes { // if package includes only imported files, then reject @@ -603,7 +571,7 @@ func (t *transitiveClosure) excludeType( } } if onlyImported { - return fmt.Errorf("filtering by type %q: %w", typeName, ErrImageFilterTypeIsImport) + return fmt.Errorf("exclusion of type %q: %w", typeName, ErrImageFilterTypeIsImport) } } // Exclude the package and all of its files. @@ -628,7 +596,6 @@ func (t *transitiveClosure) excludeElement( } return nil } - t.elements[descriptor] = inclusionModeExcluded switch descriptor := descriptor.(type) { case *descriptorpb.FileDescriptorProto: @@ -678,9 +645,15 @@ func (t *transitiveClosure) excludeElement( } } case *descriptorpb.MethodDescriptorProto: + case *descriptorpb.FieldDescriptorProto: + if descriptor.GetExtendee() == "" { + return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) + } + // Is an extension field. default: return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) } + t.elements[descriptor] = inclusionModeExcluded return nil } @@ -837,10 +810,12 @@ func (t *transitiveClosure) exploreCustomOptions( return fmt.Errorf("unexpected type for exploring options %T", descriptor) } + count := 0 optionsName := options.Descriptor().FullName() var err error options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - if !t.hasOption(fd, opts) { + count++ + if !t.hasOption(fd, imageIndex, opts) { return true } // If the value contains an Any message, we should add the message type @@ -1019,8 +994,6 @@ type imageFilterOptions struct { mutateInPlace bool includeTypes map[string]struct{} excludeTypes map[string]struct{} - includeOptions map[string]struct{} - excludeOptions map[string]struct{} } func newImageFilterOptions() *imageFilterOptions { diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 3fd1d3045f..4753db9aef 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -154,35 +154,19 @@ func TestNesting(t *testing.T) { func TestOptions(t *testing.T) { t.Parallel() - t.Run("include_jstype", func(t *testing.T) { + t.Run("include_option", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", "options.jstype.include.txtar", WithIncludeOptions("google.protobuf.FieldOptions.jstype")) - WithIncludeOptions("google.protobuf.FieldOptions.jstype") - }) - t.Run("exclude_jstype", func(t *testing.T) { - t.Parallel() - runDiffTest(t, "testdata/options", "options.jstype.exclude.txtar", WithExcludeOptions("google.protobuf.FieldOptions.jstype")) - }) - t.Run("include_message", func(t *testing.T) { - t.Parallel() - runDiffTest(t, "testdata/options", "options.message.include.txtar", WithIncludeOptions("message_foo", "message_baz")) + // Including an option is an error. + ctx := context.Background() + _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/options", bufimage.WithExcludeSourceCodeInfo()) + require.NoError(t, err) + _, err = FilterImage(image, WithIncludeTypes("message_foo")) + require.Error(t, err) + require.ErrorContains(t, err, "extension") }) t.Run("exclude_foo", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", "options.foo.exclude.txtar", WithExcludeOptions( - "message_foo", - "field_foo", - "oneof_foo", - "enum_foo", - "enum_value_foo", - "service_foo", - "method_foo", - "UsedOption.file_foo", - )) - }) - t.Run("include_foo", func(t *testing.T) { - t.Parallel() - runDiffTest(t, "testdata/options", "options.foo.include.txtar", WithIncludeOptions( + runDiffTest(t, "testdata/options", "options.foo.exclude.txtar", WithExcludeTypes( "message_foo", "field_foo", "oneof_foo", @@ -195,7 +179,7 @@ func TestOptions(t *testing.T) { }) t.Run("only_file", func(t *testing.T) { t.Parallel() - runDiffTest(t, "testdata/options", "options.only_file.txtar", WithExcludeOptions( + runDiffTest(t, "testdata/options", "options.only_file.txtar", WithExcludeTypes( "message_foo", "message_bar", "message_baz", "field_foo", "field_bar", "field_baz", "oneof_foo", "oneof_bar", "oneof_baz", @@ -236,24 +220,19 @@ func TestOptionImports(t *testing.T) { ) require.NoError(t, err) - t.Run("none_excluded", func(t *testing.T) { - t.Parallel() - generated := runFilterImage(t, image, WithExcludeOptions("google.protobuf.FieldOptions.jstype")) - checkExpectation(t, context.Background(), generated, bucket, "none_excluded.txtar") - }) t.Run("exclude_foo", func(t *testing.T) { t.Parallel() - generated := runFilterImage(t, image, WithExcludeOptions("message_foo")) + generated := runFilterImage(t, image, WithExcludeTypes("message_foo")) checkExpectation(t, context.Background(), generated, bucket, "foo.txtar") }) t.Run("exclude_foo_bar", func(t *testing.T) { t.Parallel() - generated := runFilterImage(t, image, WithExcludeOptions("message_foo", "message_bar")) + generated := runFilterImage(t, image, WithExcludeTypes("message_foo", "message_bar")) checkExpectation(t, context.Background(), generated, bucket, "foo_bar.txtar") }) t.Run("exclude_bar", func(t *testing.T) { t.Parallel() - generated := runFilterImage(t, image, WithIncludeOptions("message_foo"), WithExcludeOptions("message_bar")) + generated := runFilterImage(t, image, WithIncludeTypes("pkg.Foo"), WithExcludeTypes("message_bar")) checkExpectation(t, context.Background(), generated, bucket, "bar.txtar") }) } diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 19f93734c3..b790ebefea 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -549,7 +549,7 @@ func (b *sourcePathsBuilder) remapOptions( options := optionsMessage.ProtoReflect() numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - if !b.closure.hasOption(fd, b.options) { + if !b.closure.hasOption(fd, b.imageIndex, b.options) { // Remove this option. optionPath := append(optionsPath, int32(fd.Number())) sourcePathsRemap.markDeleted(optionPath) diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar index bbf49af0f9..ed614ef216 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/imports/bar.txtar @@ -12,86 +12,63 @@ message Foo { -- google/protobuf/descriptor.proto -- syntax = "proto2"; package google.protobuf; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2; - optional VerificationState verification = 3 [default = UNVERIFIED]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; message FeatureSet { - optional FieldPresence field_presence = 1; - optional EnumType enum_type = 2; - optional RepeatedFieldEncoding repeated_field_encoding = 3; - optional Utf8Validation utf8_validation = 4; - optional MessageEncoding message_encoding = 5; - optional JsonFormat json_format = 6; + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; enum EnumType { ENUM_TYPE_UNKNOWN = 0; OPEN = 1; @@ -124,241 +101,42 @@ message FeatureSet { NONE = 3; reserved 1; } - extensions 1000 to 9994, 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" } - } + ]; + extensions 9995 to 9999, 10000; + reserved 999; } message MessageOptions { optional bool message_set_wire_format = 1 [default = false]; optional bool no_standard_descriptor_accessor = 2 [default = false]; optional bool deprecated = 3 [default = false]; optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; optional FeatureSet features = 12; repeated UninterpretedOption uninterpreted_option = 999; extensions 1000 to max; reserved 4, 5, 6, 8, 9; } -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1; - repeated int32 span = 2; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000; -} message UninterpretedOption { repeated NamePart name = 2; optional string identifier_value = 3; @@ -372,30 +150,12 @@ message UninterpretedOption { required bool is_extension = 2; } } -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} -- options.proto -- syntax = "proto3"; import "google/protobuf/descriptor.proto"; -message OptionBar { - string bar = 1; -} message OptionFoo { string foo = 1; } extend google.protobuf.MessageOptions { OptionFoo message_foo = 50000; - OptionBar message_bar = 50001; } diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar index 0706f98c00..68c301a0f9 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/imports/foo.txtar @@ -480,6 +480,5 @@ message OptionFoo { string foo = 1; } extend google.protobuf.MessageOptions { - OptionFoo message_foo = 50000; OptionBar message_bar = 50001; } diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar deleted file mode 100644 index 1003545b2f..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/imports/none_excluded.txtar +++ /dev/null @@ -1,486 +0,0 @@ --- a.proto -- -syntax = "proto3"; -package pkg; -import "options.proto"; -message Foo { - option (message_foo) = { foo: "str" }; - Bar nested_bar = 2; - message Bar { - option (message_bar) = { bar: "str" }; - string bar = 1; - } -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -option cc_enable_arenas = true; -option csharp_namespace = "Google.Protobuf.Reflection"; -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_outer_classname = "DescriptorProtos"; -option java_package = "com.google.protobuf"; -option objc_class_prefix = "GPB"; -option optimize_for = SPEED; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; - optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1 [ - edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, - edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, - edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional EnumType enum_type = 2 [ - edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, - edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - optional RepeatedFieldEncoding repeated_field_encoding = 3 [ - edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, - edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional Utf8Validation utf8_validation = 4 [ - edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, - edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional MessageEncoding message_encoding = 5 [ - edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional JsonFormat json_format = 6 [ - edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, - edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_MESSAGE, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994 [ - declaration = { - number: 1000, - full_name: ".pb.cpp", - type: ".pb.CppFeatures" - }, - declaration = { - number: 1001, - full_name: ".pb.java", - type: ".pb.JavaFeatures" - }, - declaration = { - number: 1002, - full_name: ".pb.go", - type: ".pb.GoFeatures" - }, - declaration = { - number: 9990, - full_name: ".pb.proto1", - type: ".pb.Proto1Features" - } - ]; - extensions 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", - type: ".buf.descriptor.v1.FileDescriptorSetExtension" - } - ]; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20 [deprecated = true]; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1 [packed = true]; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1 [packed = true]; - repeated int32 span = 2 [packed = true]; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_source_code_info_extension", - type: ".buf.descriptor.v1.SourceCodeInfoExtension" - } - ]; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message OptionBar { - string bar = 1; -} -message OptionFoo { - string foo = 1; -} -extend google.protobuf.MessageOptions { - OptionFoo message_foo = 50000; - OptionBar message_bar = 50001; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar index 384eaf7a15..92c977db6b 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.exclude.txtar @@ -506,43 +506,35 @@ message UnusedOption { message UsedOption { string foo = 1; extend google.protobuf.FileOptions { - UsedOption file_foo = 50000; UnusedOption file_bar = 50001; string file_baz = 50002; } } extend google.protobuf.EnumOptions { - UsedOption enum_foo = 50000; UnusedOption enum_bar = 50001; string enum_baz = 50002; } extend google.protobuf.EnumValueOptions { - UsedOption enum_value_foo = 50000; UnusedOption enum_value_bar = 50001; string enum_value_baz = 50002; } extend google.protobuf.FieldOptions { - UsedOption field_foo = 50000; UnusedOption field_bar = 50001; string field_baz = 50002; } extend google.protobuf.MessageOptions { - UsedOption message_foo = 50000; UnusedOption message_bar = 50001; string message_baz = 50002; } extend google.protobuf.MethodOptions { - UsedOption method_foo = 50000; UnusedOption method_bar = 50001; string method_baz = 50002; } extend google.protobuf.OneofOptions { - UsedOption oneof_foo = 50000; UnusedOption oneof_bar = 50001; string oneof_baz = 50002; } extend google.protobuf.ServiceOptions { - UsedOption service_foo = 50000; UnusedOption service_bar = 50001; string service_baz = 50002; } diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar deleted file mode 100644 index 460b41c801..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar +++ /dev/null @@ -1,463 +0,0 @@ --- a.proto -- -syntax = "proto2"; -package pkg; -import "options.proto"; -option (UsedOption.file_foo) = { foo: "str" }; -message Empty { -} -message Foo { - option (message_foo) = { foo: "str" }; - optional uint64 foo = 1 [(field_foo) = { foo: "str" }]; - oneof testOneof { - option (oneof_foo) = { foo: "str" }; - string bar = 2; - bytes baz = 3; - } - extensions 10 to max; -} -enum FooEnum { - option (enum_foo) = { foo: "str" }; - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1 [(enum_value_foo) = { foo: "str" }]; -} -service FooService { - option (service_foo) = { foo: "str" }; - rpc Do ( Empty ) returns ( Empty ) { - option (method_foo) = { foo: "str" }; - } - rpc DoNot ( Empty ) returns ( Empty ) { - option (method_foo) = { foo: "str" }; - } -} -extend Foo { - optional string extension = 11 [(field_foo) = { foo: "str" }]; -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2; - optional VerificationState verification = 3 [default = UNVERIFIED]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1; - optional EnumType enum_type = 2; - optional RepeatedFieldEncoding repeated_field_encoding = 3; - optional Utf8Validation utf8_validation = 4; - optional MessageEncoding message_encoding = 5; - optional JsonFormat json_format = 6; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994, 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1; - repeated int32 span = 2; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message Files { - google.protobuf.FileDescriptorSet files = 1; -} -message UnusedOption { - string foo = 1; -} -message UsedOption { - string foo = 1; - extend google.protobuf.FileOptions { - UsedOption file_foo = 50000; - UnusedOption file_bar = 50001; - string file_baz = 50002; - } -} -extend google.protobuf.EnumOptions { - UsedOption enum_foo = 50000; - UnusedOption enum_bar = 50001; - string enum_baz = 50002; -} -extend google.protobuf.EnumValueOptions { - UsedOption enum_value_foo = 50000; - UnusedOption enum_value_bar = 50001; - string enum_value_baz = 50002; -} -extend google.protobuf.FieldOptions { - UsedOption field_foo = 50000; - UnusedOption field_bar = 50001; - string field_baz = 50002; -} -extend google.protobuf.MessageOptions { - UsedOption message_foo = 50000; - UnusedOption message_bar = 50001; - string message_baz = 50002; -} -extend google.protobuf.MethodOptions { - UsedOption method_foo = 50000; - UnusedOption method_bar = 50001; - string method_baz = 50002; -} -extend google.protobuf.OneofOptions { - UsedOption oneof_foo = 50000; - UnusedOption oneof_bar = 50001; - string oneof_baz = 50002; -} -extend google.protobuf.ServiceOptions { - UsedOption service_foo = 50000; - UnusedOption service_bar = 50001; - string service_baz = 50002; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar deleted file mode 100644 index d1f0974c84..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.exclude.txtar +++ /dev/null @@ -1,558 +0,0 @@ --- a.proto -- -syntax = "proto2"; -package pkg; -import "options.proto"; -option (UsedOption.file_baz) = "str"; -option (UsedOption.file_foo) = { foo: "str" }; -message Empty { -} -message Foo { - option (message_baz) = "str"; - option (message_foo) = { foo: "str" }; - optional uint64 foo = 1 [(field_baz) = "str", (field_foo) = { foo: "str" }]; - oneof testOneof { - option (oneof_baz) = "str"; - option (oneof_foo) = { foo: "str" }; - string bar = 2; - bytes baz = 3; - } - extensions 10 to max; -} -enum FooEnum { - option deprecated = true; - option (enum_baz) = "str"; - option (enum_foo) = { foo: "str" }; - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1 [ - (enum_value_baz) = "str", - (enum_value_foo) = { foo: "str" } - ]; -} -service FooService { - option (service_baz) = "str"; - option (service_foo) = { foo: "str" }; - rpc Do ( Empty ) returns ( Empty ) { - option (method_baz) = "str"; - option (method_foo) = { foo: "str" }; - } - rpc DoNot ( Empty ) returns ( Empty ) { - option (method_baz) = "str"; - option (method_foo) = { foo: "str" }; - } -} -extend Foo { - optional string extension = 11 [(field_baz) = "str", (field_foo) = { foo: "str" }]; -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -option cc_enable_arenas = true; -option csharp_namespace = "Google.Protobuf.Reflection"; -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_outer_classname = "DescriptorProtos"; -option java_package = "com.google.protobuf"; -option objc_class_prefix = "GPB"; -option optimize_for = SPEED; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; - optional VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1 [ - edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, - edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, - edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional EnumType enum_type = 2 [ - edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, - edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - optional RepeatedFieldEncoding repeated_field_encoding = 3 [ - edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, - edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional Utf8Validation utf8_validation = 4 [ - edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, - edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional MessageEncoding message_encoding = 5 [ - edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_FIELD, - targets = TARGET_TYPE_FILE - ]; - optional JsonFormat json_format = 6 [ - edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, - edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, - feature_support = { edition_introduced: EDITION_2023 }, - retention = RETENTION_RUNTIME, - targets = TARGET_TYPE_MESSAGE, - targets = TARGET_TYPE_ENUM, - targets = TARGET_TYPE_FILE - ]; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994 [ - declaration = { - number: 1000, - full_name: ".pb.cpp", - type: ".pb.CppFeatures" - }, - declaration = { - number: 1001, - full_name: ".pb.java", - type: ".pb.JavaFeatures" - }, - declaration = { - number: 1002, - full_name: ".pb.go", - type: ".pb.GoFeatures" - }, - declaration = { - number: 9990, - full_name: ".pb.proto1", - type: ".pb.Proto1Features" - } - ]; - extensions 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_file_descriptor_set_extension", - type: ".buf.descriptor.v1.FileDescriptorSetExtension" - } - ]; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20 [deprecated = true]; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1 [packed = true]; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1 [packed = true]; - repeated int32 span = 2 [packed = true]; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000 [ - declaration = { - number: 536000000, - full_name: ".buf.descriptor.v1.buf_source_code_info_extension", - type: ".buf.descriptor.v1.SourceCodeInfoExtension" - } - ]; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message Files { - google.protobuf.FileDescriptorSet files = 1; -} -message UnusedOption { - string foo = 1; -} -message UsedOption { - string foo = 1; - extend google.protobuf.FileOptions { - UsedOption file_foo = 50000; - UnusedOption file_bar = 50001; - string file_baz = 50002; - } -} -extend google.protobuf.EnumOptions { - UsedOption enum_foo = 50000; - UnusedOption enum_bar = 50001; - string enum_baz = 50002; -} -extend google.protobuf.EnumValueOptions { - UsedOption enum_value_foo = 50000; - UnusedOption enum_value_bar = 50001; - string enum_value_baz = 50002; -} -extend google.protobuf.FieldOptions { - UsedOption field_foo = 50000; - UnusedOption field_bar = 50001; - string field_baz = 50002; -} -extend google.protobuf.MessageOptions { - UsedOption message_foo = 50000; - UnusedOption message_bar = 50001; - string message_baz = 50002; -} -extend google.protobuf.MethodOptions { - UsedOption method_foo = 50000; - UnusedOption method_bar = 50001; - string method_baz = 50002; -} -extend google.protobuf.OneofOptions { - UsedOption oneof_foo = 50000; - UnusedOption oneof_bar = 50001; - string oneof_baz = 50002; -} -extend google.protobuf.ServiceOptions { - UsedOption service_foo = 50000; - UnusedOption service_bar = 50001; - string service_baz = 50002; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar deleted file mode 100644 index 95f4d7a12b..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.jstype.include.txtar +++ /dev/null @@ -1,453 +0,0 @@ --- a.proto -- -syntax = "proto2"; -package pkg; -message Empty { -} -message Foo { - optional uint64 foo = 1 [jstype = JS_STRING]; - oneof testOneof { - string bar = 2; - bytes baz = 3; - } - extensions 10 to max; -} -enum FooEnum { - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1; -} -service FooService { - rpc Do ( Empty ) returns ( Empty ); - rpc DoNot ( Empty ) returns ( Empty ); -} -extend Foo { - optional string extension = 11; -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2; - optional VerificationState verification = 3 [default = UNVERIFIED]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1; - optional EnumType enum_type = 2; - optional RepeatedFieldEncoding repeated_field_encoding = 3; - optional Utf8Validation utf8_validation = 4; - optional MessageEncoding message_encoding = 5; - optional JsonFormat json_format = 6; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994, 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1; - repeated int32 span = 2; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message Files { - google.protobuf.FileDescriptorSet files = 1; -} -message UnusedOption { - string foo = 1; -} -message UsedOption { - string foo = 1; - extend google.protobuf.FileOptions { - UsedOption file_foo = 50000; - UnusedOption file_bar = 50001; - string file_baz = 50002; - } -} -extend google.protobuf.EnumOptions { - UsedOption enum_foo = 50000; - UnusedOption enum_bar = 50001; - string enum_baz = 50002; -} -extend google.protobuf.EnumValueOptions { - UsedOption enum_value_foo = 50000; - UnusedOption enum_value_bar = 50001; - string enum_value_baz = 50002; -} -extend google.protobuf.FieldOptions { - UsedOption field_foo = 50000; - UnusedOption field_bar = 50001; - string field_baz = 50002; -} -extend google.protobuf.MessageOptions { - UsedOption message_foo = 50000; - UnusedOption message_bar = 50001; - string message_baz = 50002; -} -extend google.protobuf.MethodOptions { - UsedOption method_foo = 50000; - UnusedOption method_bar = 50001; - string method_baz = 50002; -} -extend google.protobuf.OneofOptions { - UsedOption oneof_foo = 50000; - UnusedOption oneof_bar = 50001; - string oneof_baz = 50002; -} -extend google.protobuf.ServiceOptions { - UsedOption service_foo = 50000; - UnusedOption service_bar = 50001; - string service_baz = 50002; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar deleted file mode 100644 index 28352e873c..0000000000 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.message.include.txtar +++ /dev/null @@ -1,456 +0,0 @@ --- a.proto -- -syntax = "proto2"; -package pkg; -import "options.proto"; -message Empty { -} -message Foo { - option (message_baz) = "str"; - option (message_foo) = { foo: "str" }; - optional uint64 foo = 1; - oneof testOneof { - string bar = 2; - bytes baz = 3; - } - extensions 10 to max; -} -enum FooEnum { - FOO_ENUM_X = 0; - FOO_ENUM_Y = 1; -} -service FooService { - rpc Do ( Empty ) returns ( Empty ); - rpc DoNot ( Empty ) returns ( Empty ); -} -extend Foo { - optional string extension = 11; -} --- google/protobuf/descriptor.proto -- -syntax = "proto2"; -package google.protobuf; -message DescriptorProto { - optional string name = 1; - repeated FieldDescriptorProto field = 2; - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - repeated ExtensionRange extension_range = 5; - repeated FieldDescriptorProto extension = 6; - optional MessageOptions options = 7; - repeated OneofDescriptorProto oneof_decl = 8; - repeated ReservedRange reserved_range = 9; - repeated string reserved_name = 10; - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - optional ExtensionRangeOptions options = 3; - } - message ReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumDescriptorProto { - optional string name = 1; - repeated EnumValueDescriptorProto value = 2; - optional EnumOptions options = 3; - repeated EnumReservedRange reserved_range = 4; - repeated string reserved_name = 5; - message EnumReservedRange { - optional int32 start = 1; - optional int32 end = 2; - } -} -message EnumOptions { - optional bool allow_alias = 2; - optional bool deprecated = 3 [default = false]; - optional bool deprecated_legacy_json_field_conflicts = 6; - optional FeatureSet features = 7; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 5; -} -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - optional EnumValueOptions options = 3; -} -message EnumValueOptions { - optional bool deprecated = 1 [default = false]; - optional FeatureSet features = 2; - optional bool debug_redact = 3 [default = false]; - optional FieldOptions.FeatureSupport feature_support = 4; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ExtensionRangeOptions { - repeated Declaration declaration = 2; - optional VerificationState verification = 3 [default = UNVERIFIED]; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - message Declaration { - optional int32 number = 1; - optional string full_name = 2; - optional string type = 3; - optional bool reserved = 5; - optional bool repeated = 6; - reserved 4; - } - enum VerificationState { - DECLARATION = 0; - UNVERIFIED = 1; - } - extensions 1000 to max; -} -message FeatureSet { - optional FieldPresence field_presence = 1; - optional EnumType enum_type = 2; - optional RepeatedFieldEncoding repeated_field_encoding = 3; - optional Utf8Validation utf8_validation = 4; - optional MessageEncoding message_encoding = 5; - optional JsonFormat json_format = 6; - enum EnumType { - ENUM_TYPE_UNKNOWN = 0; - OPEN = 1; - CLOSED = 2; - } - enum FieldPresence { - FIELD_PRESENCE_UNKNOWN = 0; - EXPLICIT = 1; - IMPLICIT = 2; - LEGACY_REQUIRED = 3; - } - enum JsonFormat { - JSON_FORMAT_UNKNOWN = 0; - ALLOW = 1; - LEGACY_BEST_EFFORT = 2; - } - enum MessageEncoding { - MESSAGE_ENCODING_UNKNOWN = 0; - LENGTH_PREFIXED = 1; - DELIMITED = 2; - } - enum RepeatedFieldEncoding { - REPEATED_FIELD_ENCODING_UNKNOWN = 0; - PACKED = 1; - EXPANDED = 2; - } - enum Utf8Validation { - UTF8_VALIDATION_UNKNOWN = 0; - VERIFY = 2; - NONE = 3; - reserved 1; - } - extensions 1000 to 9994, 9995 to 9999, 10000; - reserved 999; -} -message FeatureSetDefaults { - repeated FeatureSetEditionDefault defaults = 1; - optional Edition minimum_edition = 4; - optional Edition maximum_edition = 5; - message FeatureSetEditionDefault { - optional Edition edition = 3; - optional FeatureSet overridable_features = 4; - optional FeatureSet fixed_features = 5; - reserved 1, 2; - reserved "features"; - } -} -message FieldDescriptorProto { - optional string name = 1; - optional string extendee = 2; - optional int32 number = 3; - optional Label label = 4; - optional Type type = 5; - optional string type_name = 6; - optional string default_value = 7; - optional FieldOptions options = 8; - optional int32 oneof_index = 9; - optional string json_name = 10; - optional bool proto3_optional = 17; - enum Label { - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - enum Type { - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; - TYPE_SINT64 = 18; - } -} -message FieldOptions { - optional CType ctype = 1 [default = STRING]; - optional bool packed = 2; - optional bool deprecated = 3 [default = false]; - optional bool lazy = 5 [default = false]; - optional JSType jstype = 6 [default = JS_NORMAL]; - optional bool weak = 10 [default = false]; - optional bool unverified_lazy = 15 [default = false]; - optional bool debug_redact = 16 [default = false]; - optional OptionRetention retention = 17; - repeated OptionTargetType targets = 19; - repeated EditionDefault edition_defaults = 20; - optional FeatureSet features = 21; - optional FeatureSupport feature_support = 22; - repeated UninterpretedOption uninterpreted_option = 999; - message EditionDefault { - optional string value = 2; - optional Edition edition = 3; - } - message FeatureSupport { - optional Edition edition_introduced = 1; - optional Edition edition_deprecated = 2; - optional string deprecation_warning = 3; - optional Edition edition_removed = 4; - } - enum CType { - STRING = 0; - CORD = 1; - STRING_PIECE = 2; - } - enum JSType { - JS_NORMAL = 0; - JS_STRING = 1; - JS_NUMBER = 2; - } - enum OptionRetention { - RETENTION_UNKNOWN = 0; - RETENTION_RUNTIME = 1; - RETENTION_SOURCE = 2; - } - enum OptionTargetType { - TARGET_TYPE_UNKNOWN = 0; - TARGET_TYPE_FILE = 1; - TARGET_TYPE_EXTENSION_RANGE = 2; - TARGET_TYPE_MESSAGE = 3; - TARGET_TYPE_FIELD = 4; - TARGET_TYPE_ONEOF = 5; - TARGET_TYPE_ENUM = 6; - TARGET_TYPE_ENUM_ENTRY = 7; - TARGET_TYPE_SERVICE = 8; - TARGET_TYPE_METHOD = 9; - } - extensions 1000 to max; - reserved 4, 18; -} -message FileDescriptorProto { - optional string name = 1; - optional string package = 2; - repeated string dependency = 3; - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - optional FileOptions options = 8; - optional SourceCodeInfo source_code_info = 9; - repeated int32 public_dependency = 10; - repeated int32 weak_dependency = 11; - optional string syntax = 12; - optional Edition edition = 14; -} -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; - extensions 536000000; -} -message FileOptions { - optional string java_package = 1; - optional string java_outer_classname = 8; - optional OptimizeMode optimize_for = 9 [default = SPEED]; - optional bool java_multiple_files = 10 [default = false]; - optional string go_package = 11; - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool java_generate_equals_and_hash = 20; - optional bool deprecated = 23 [default = false]; - optional bool java_string_check_utf8 = 27 [default = false]; - optional bool cc_enable_arenas = 31 [default = true]; - optional string objc_class_prefix = 36; - optional string csharp_namespace = 37; - optional string swift_prefix = 39; - optional string php_class_prefix = 40; - optional string php_namespace = 41; - optional string php_metadata_namespace = 44; - optional string ruby_package = 45; - optional FeatureSet features = 50; - repeated UninterpretedOption uninterpreted_option = 999; - enum OptimizeMode { - SPEED = 1; - CODE_SIZE = 2; - LITE_RUNTIME = 3; - } - extensions 1000 to max; - reserved 38, 42; - reserved "php_generic_services"; -} -message GeneratedCodeInfo { - repeated Annotation annotation = 1; - message Annotation { - repeated int32 path = 1; - optional string source_file = 2; - optional int32 begin = 3; - optional int32 end = 4; - optional Semantic semantic = 5; - enum Semantic { - NONE = 0; - SET = 1; - ALIAS = 2; - } - } -} -message MessageOptions { - optional bool message_set_wire_format = 1 [default = false]; - optional bool no_standard_descriptor_accessor = 2 [default = false]; - optional bool deprecated = 3 [default = false]; - optional bool map_entry = 7; - optional bool deprecated_legacy_json_field_conflicts = 11; - optional FeatureSet features = 12; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; - reserved 4, 5, 6, 8, 9; -} -message MethodDescriptorProto { - optional string name = 1; - optional string input_type = 2; - optional string output_type = 3; - optional MethodOptions options = 4; - optional bool client_streaming = 5 [default = false]; - optional bool server_streaming = 6 [default = false]; -} -message MethodOptions { - optional bool deprecated = 33 [default = false]; - optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; - optional FeatureSet features = 35; - repeated UninterpretedOption uninterpreted_option = 999; - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; - IDEMPOTENT = 2; - } - extensions 1000 to max; -} -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} -message OneofOptions { - optional FeatureSet features = 1; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - optional ServiceOptions options = 3; -} -message ServiceOptions { - optional bool deprecated = 33 [default = false]; - optional FeatureSet features = 34; - repeated UninterpretedOption uninterpreted_option = 999; - extensions 1000 to max; -} -message SourceCodeInfo { - repeated Location location = 1; - message Location { - repeated int32 path = 1; - repeated int32 span = 2; - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } - extensions 536000000; -} -message UninterpretedOption { - repeated NamePart name = 2; - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } -} -enum Edition { - EDITION_UNKNOWN = 0; - EDITION_1_TEST_ONLY = 1; - EDITION_2_TEST_ONLY = 2; - EDITION_LEGACY = 900; - EDITION_PROTO2 = 998; - EDITION_PROTO3 = 999; - EDITION_2023 = 1000; - EDITION_2024 = 1001; - EDITION_99997_TEST_ONLY = 99997; - EDITION_99998_TEST_ONLY = 99998; - EDITION_99999_TEST_ONLY = 99999; - EDITION_MAX = 2147483647; -} --- options.proto -- -syntax = "proto3"; -import "google/protobuf/descriptor.proto"; -message Files { - google.protobuf.FileDescriptorSet files = 1; -} -message UnusedOption { - string foo = 1; -} -message UsedOption { - string foo = 1; - extend google.protobuf.FileOptions { - UsedOption file_foo = 50000; - UnusedOption file_bar = 50001; - string file_baz = 50002; - } -} -extend google.protobuf.EnumOptions { - UsedOption enum_foo = 50000; - UnusedOption enum_bar = 50001; - string enum_baz = 50002; -} -extend google.protobuf.EnumValueOptions { - UsedOption enum_value_foo = 50000; - UnusedOption enum_value_bar = 50001; - string enum_value_baz = 50002; -} -extend google.protobuf.FieldOptions { - UsedOption field_foo = 50000; - UnusedOption field_bar = 50001; - string field_baz = 50002; -} -extend google.protobuf.MessageOptions { - UsedOption message_foo = 50000; - UnusedOption message_bar = 50001; - string message_baz = 50002; -} -extend google.protobuf.MethodOptions { - UsedOption method_foo = 50000; - UnusedOption method_bar = 50001; - string method_baz = 50002; -} -extend google.protobuf.OneofOptions { - UsedOption oneof_foo = 50000; - UnusedOption oneof_bar = 50001; - string oneof_baz = 50002; -} -extend google.protobuf.ServiceOptions { - UsedOption service_foo = 50000; - UnusedOption service_bar = 50001; - string service_baz = 50002; -} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar index 1f159f9e12..9b02187490 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.only_file.txtar @@ -504,38 +504,3 @@ message UsedOption { string file_baz = 50002; } } -extend google.protobuf.EnumOptions { - UsedOption enum_foo = 50000; - UnusedOption enum_bar = 50001; - string enum_baz = 50002; -} -extend google.protobuf.EnumValueOptions { - UsedOption enum_value_foo = 50000; - UnusedOption enum_value_bar = 50001; - string enum_value_baz = 50002; -} -extend google.protobuf.FieldOptions { - UsedOption field_foo = 50000; - UnusedOption field_bar = 50001; - string field_baz = 50002; -} -extend google.protobuf.MessageOptions { - UsedOption message_foo = 50000; - UnusedOption message_bar = 50001; - string message_baz = 50002; -} -extend google.protobuf.MethodOptions { - UsedOption method_foo = 50000; - UnusedOption method_bar = 50001; - string method_baz = 50002; -} -extend google.protobuf.OneofOptions { - UsedOption oneof_foo = 50000; - UnusedOption oneof_bar = 50001; - string oneof_baz = 50002; -} -extend google.protobuf.ServiceOptions { - UsedOption service_foo = 50000; - UnusedOption service_bar = 50001; - string service_baz = 50002; -} From 07ea2e02a3dc6d500f4cc05133564d7d8c1ef240 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 10:04:51 +0000 Subject: [PATCH 29/80] Docs --- .../bufimage/bufimageutil/bufimageutil.go | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index e9ac26b853..74f7a15f78 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -117,13 +117,13 @@ func WithIncludeTypes(typeNames ...string) ImageFilterOption { } } -// WithExcludeTypes returns an option for ImageFilteredByTypesWithOptions that specifies -// the set of types that should be excluded from the filtered image. +// WithExcludeTypes returns an option for ImageFilteredByTypesWithOptions that +// specifies the set of types that should be excluded from the filtered image. // // May be provided multiple times. The type names should be fully qualified. -// For example, "google.protobuf.Any" or "buf.validate". Type or package names -// are accepted. If the type does not exist in the image, an error -// [ErrImageFilterTypeNotFound] will be returned. +// For example, "google.protobuf.Any" or "buf.validate". Type, package names, +// or extension names are accepted. If the type does not exist in the image, an +// error [ErrImageFilterTypeNotFound] will be returned. func WithExcludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if len(typeNames) > 0 && opts.excludeTypes == nil { @@ -155,14 +155,15 @@ func WithMutateInPlace() ImageFilterOption { // required to define the specified types will be included in the filtered image. // // Excluded types and options are not included in the filtered image. If an -// included type transitively depens on the excluded type, the descriptor will +// included type transitively depends on the excluded type, the descriptor will // be altered to remove the dependency. // // This returns a new [bufimage.Image] that is a shallow copy of the underlying // [descriptorpb.FileDescriptorProto]s of the original. The new image may // therefore share state with the original image, so it should not be modified. -// If the original image is no longer needed, it should be discarded. To avoid -// this sharing, use the [WithMutateInPlace] option. +// If the original image is no longer needed, it should be discarded. To mutate +// the original image, use the [WithMutateInPlace] option. Otherwise, to avoid +// sharing of state clone the image before filtering. // // A descriptor is said to require another descriptor if the dependent // descriptor is needed to accurately and completely describe that descriptor. @@ -271,21 +272,21 @@ type closureInclusionMode int const ( // Element is explicitly excluded from the closure. - inclusionModeExcluded = closureInclusionMode(iota - 1) // -1 + inclusionModeExcluded = closureInclusionMode(iota - 1) // Element is not yet known to be included or excluded. - inclusionModeUnknown // 0 + inclusionModeUnknown // Element is included in closure because it is directly reachable from a root. - inclusionModeExplicit // 1 + inclusionModeExplicit // Element is included in closure because it is a message or service that // *contains* an explicitly included element but is not itself directly // reachable. - inclusionModeEnclosing // 2 + inclusionModeEnclosing // Element is included in closure because it is implied by the presence of a // custom option. For example, a field element with a custom option implies // the presence of google.protobuf.FieldOptions. An option type could instead be // explicitly included if it is also directly reachable (i.e. some type in the // graph explicitly refers to the option type). - inclusionModeImplicit // 3 + inclusionModeImplicit ) func newTransitiveClosure() *transitiveClosure { From c43b74eeb25337edc04467de21d8ded58f1f5116 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 10:23:24 +0000 Subject: [PATCH 30/80] Remove inclusion of extension restriction --- .../bufimage/bufimageutil/bufimageutil.go | 16 +- .../bufimageutil/bufimageutil_test.go | 17 +- .../options/options.foo.include.txtar | 330 ++++++++++++++++++ 3 files changed, 347 insertions(+), 16 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 74f7a15f78..0031eb6316 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -103,8 +103,9 @@ func WithAllowFilterByImportedType() ImageFilterOption { // the set of types that should be included in the filtered image. // // May be provided multiple times. The type names should be fully qualified. -// For example, "google.protobuf.Any" or "buf.validate". Type or package names -// are accepted. If the type does not exist in the image, an error +// For example, "google.protobuf.Any" or "buf.validate". Types and package names +// are accepted. Types may be nested, and can be Messages, Enums, Services, Methods, +// or Extensions. If the type does not exist in the image, an error wrapping // [ErrImageFilterTypeNotFound] will be returned. func WithIncludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { @@ -121,9 +122,10 @@ func WithIncludeTypes(typeNames ...string) ImageFilterOption { // specifies the set of types that should be excluded from the filtered image. // // May be provided multiple times. The type names should be fully qualified. -// For example, "google.protobuf.Any" or "buf.validate". Type, package names, -// or extension names are accepted. If the type does not exist in the image, an -// error [ErrImageFilterTypeNotFound] will be returned. +// For example, "google.protobuf.Any" or "buf.validate". Types and package names +// are accepted. Types may be nested, and can be Messages, Enums, Services, Methods, +// or Extensions. If the type does not exist in the image, an error wrapping +// [ErrImageFilterTypeNotFound] will be returned. func WithExcludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { if len(typeNames) > 0 && opts.excludeTypes == nil { @@ -351,10 +353,6 @@ func (t *transitiveClosure) includeType( // matching, such as ability to get a package AND all of its sub-packages. descriptorInfo, ok := imageIndex.ByName[typeName] if ok { - // If an extension field, error. Extension fields can not be included only excluded. - if field, ok := descriptorInfo.element.(*descriptorpb.FieldDescriptorProto); ok && field.GetExtendee() != "" { - return fmt.Errorf("inclusion of type %q: extension fields can not be included", typeName) - } // It's a type name if !options.allowImportedTypes && descriptorInfo.file.IsImport() { return fmt.Errorf("inclusion of type %q: %w", typeName, ErrImageFilterTypeIsImport) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 4753db9aef..c4990e7596 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -156,13 +156,16 @@ func TestOptions(t *testing.T) { t.Parallel() t.Run("include_option", func(t *testing.T) { t.Parallel() - // Including an option is an error. - ctx := context.Background() - _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/options", bufimage.WithExcludeSourceCodeInfo()) - require.NoError(t, err) - _, err = FilterImage(image, WithIncludeTypes("message_foo")) - require.Error(t, err) - require.ErrorContains(t, err, "extension") + runDiffTest(t, "testdata/options", "options.foo.include.txtar", WithIncludeTypes( + "message_foo", + "field_foo", + "oneof_foo", + "enum_foo", + "enum_value_foo", + "service_foo", + "method_foo", + "UsedOption.file_foo", + )) }) t.Run("exclude_foo", func(t *testing.T) { t.Parallel() diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar new file mode 100644 index 0000000000..8d6c038c76 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/options.foo.include.txtar @@ -0,0 +1,330 @@ +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + optional FeatureSet features = 7; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 5; +} +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + optional FeatureSet features = 2; + optional bool debug_redact = 3 [default = false]; + optional FieldOptions.FeatureSupport feature_support = 4; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + optional bool packed = 2; + optional bool deprecated = 3 [default = false]; + optional bool lazy = 5 [default = false]; + optional JSType jstype = 6 [default = JS_NORMAL]; + optional bool weak = 10 [default = false]; + optional bool unverified_lazy = 15 [default = false]; + optional bool debug_redact = 16 [default = false]; + optional OptionRetention retention = 17; + repeated OptionTargetType targets = 19; + repeated EditionDefault edition_defaults = 20; + optional FeatureSet features = 21; + optional FeatureSupport feature_support = 22; + repeated UninterpretedOption uninterpreted_option = 999; + message EditionDefault { + optional string value = 2; + optional Edition edition = 3; + } + message FeatureSupport { + optional Edition edition_introduced = 1; + optional Edition edition_deprecated = 2; + optional string deprecation_warning = 3; + optional Edition edition_removed = 4; + } + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + extensions 1000 to max; + reserved 4, 18; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + optional FeatureSet features = 12; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 4, 5, 6, 8, 9; +} +message MethodOptions { + optional bool deprecated = 33 [default = false]; + optional IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; + optional FeatureSet features = 35; + repeated UninterpretedOption uninterpreted_option = 999; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + extensions 1000 to max; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +enum Edition { + EDITION_UNKNOWN = 0; + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_LEGACY = 900; + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + EDITION_2023 = 1000; + EDITION_2024 = 1001; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + EDITION_MAX = 2147483647; +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message UnusedOption { + string foo = 1; +} +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + UnusedOption file_bar = 50001; + string file_baz = 50002; + } +} +extend google.protobuf.EnumOptions { + UsedOption enum_foo = 50000; + UnusedOption enum_bar = 50001; + string enum_baz = 50002; +} +extend google.protobuf.EnumValueOptions { + UsedOption enum_value_foo = 50000; + UnusedOption enum_value_bar = 50001; + string enum_value_baz = 50002; +} +extend google.protobuf.FieldOptions { + UsedOption field_foo = 50000; + UnusedOption field_bar = 50001; + string field_baz = 50002; +} +extend google.protobuf.MessageOptions { + UsedOption message_foo = 50000; + UnusedOption message_bar = 50001; + string message_baz = 50002; +} +extend google.protobuf.MethodOptions { + UsedOption method_foo = 50000; + UnusedOption method_bar = 50001; + string method_baz = 50002; +} +extend google.protobuf.OneofOptions { + UsedOption oneof_foo = 50000; + UnusedOption oneof_bar = 50001; + string oneof_baz = 50002; +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + UnusedOption service_bar = 50001; + string service_baz = 50002; +} From d66e197bbbb5d47a75e2bf509ca137553853d6f4 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 10:56:45 +0000 Subject: [PATCH 31/80] Add exclude-types flag --- .../buf/cmd/buf/command/generate/generate.go | 29 ++++++++++++++++++- private/bufpkg/bufconfig/input_config.go | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index d2552f9ec0..9efada6885 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -50,6 +50,7 @@ const ( disableSymlinksFlagName = "disable-symlinks" typeFlagName = "type" typeDeprecatedFlagName = "include-types" + excludeTypeFlagName = "exclude-type" ) // NewCommand returns a new Command. @@ -95,6 +96,16 @@ func NewCommand( # Whether to generate code for the well-known types. # Optional. include_wkt: false + # Include only these types for this plugin. + # Optional. + types: + - "foo.v1.User" + # Exclude these types for this plugin. + # Optional. + exclude_types: + - "buf.validate.oneof" + - "buf.validate.message" + - "buf.validate.field"" # The name of a local plugin if discoverable in "${PATH}" or its path in the file system. - local: protoc-gen-es @@ -248,6 +259,10 @@ func NewCommand( types: - "foo.v1.User" - "foo.v1.UserService" + # Exclude these types. + # Optional. + exclude_types: + - "buf.validate" # Only generate code for files in these paths. # If empty, include all paths. paths: @@ -388,6 +403,7 @@ type flags struct { // want to find out what will break if we do. Types []string TypesDeprecated []string + ExcludeTypes []string // special InputHashtag string } @@ -462,6 +478,12 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { nil, "The types (package, message, enum, extension, service, method) that should be included in this image. When specified, the resulting image will only include descriptors to describe the requested types. Flag usage overrides buf.gen.yaml", ) + flagSet.StringSliceVar( + &f.ExcludeTypes, + excludeTypeFlagName, + nil, + "The types (package, message, enum, extension, service, method) that should be excluded from this image. When specified, the resulting image will not include descriptors to describe the requested types. Flag usage overrides buf.gen.yaml", + ) _ = flagSet.MarkDeprecated(typeDeprecatedFlagName, fmt.Sprintf("use --%s instead", typeFlagName)) _ = flagSet.MarkHidden(typeDeprecatedFlagName) } @@ -517,6 +539,7 @@ func run( flags.Paths, flags.ExcludePaths, flags.Types, + flags.ExcludeTypes, ) if err != nil { return err @@ -591,6 +614,7 @@ func getInputImages( targetPathsOverride []string, excludePathsOverride []string, includeTypesOverride []string, + excludeTypesOverride []string, ) ([]bufimage.Image, error) { // If input is specified on the command line, we use that. If input is not // specified on the command line, use the default input. @@ -628,13 +652,16 @@ func getInputImages( if len(excludePathsOverride) > 0 { excludePaths = excludePathsOverride } - // In V2 we do not need to look at generateTypeConfig.IncludeTypes() + // In V1 we do not need to look at generateTypeConfig.IncludeTypes() // because it is always nil. includeTypes := inputConfig.IncludeTypes() if len(includeTypesOverride) > 0 { includeTypes = includeTypesOverride } excludeTypes := inputConfig.ExcludeTypes() + if len(excludeTypesOverride) > 0 { + excludeTypes = excludeTypesOverride + } inputImage, err := controller.GetImageForInputConfig( ctx, inputConfig, diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index aeb7a57079..62e8bca7df 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -406,6 +406,7 @@ func newInputConfigFromExternalV2(externalConfig externalInputConfigV2) (InputCo inputConfig.inputConfigType = inputConfigType // Types, TargetPaths, and ExcludePaths. inputConfig.includeTypes = externalConfig.Types + inputConfig.excludeTypes = externalConfig.ExcludeTypes inputConfig.targetPaths = externalConfig.TargetPaths inputConfig.excludePaths = externalConfig.ExcludePaths // Options depending on input format. From 883527c7dc521c7466818a21356159ef49552776 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 14:13:00 +0000 Subject: [PATCH 32/80] Add CHANGELOG --- CHANGELOG.md | 3 ++ .../bufimage/bufimageutil/bufimageutil.go | 34 +++++++------------ .../bufimageutil/bufimageutil_test.go | 2 +- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b65552b6d1..957f6feb96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## [Unreleased] - Fix `buf convert` to allow for zero length for `binpb`, `txtpb`, and `yaml` formats. +- Improved type filtering for `buf generate`. Adds the ability to exclude types with the parameter + `exclude_types` in `buf.gen.yaml` and a flag `--exclude-types` in the CLI. + Type filters may now also be specified as plugin parameters in `buf.gen.yaml`. ## [v1.50.1] - 2025-03-10 diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 0031eb6316..3a5c8f922a 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -90,10 +90,11 @@ func WithExcludeKnownExtensions() ImageFilterOption { } } -// WithAllowFilterByImportedType returns an option for ImageFilteredByTypesWithOptions -// that allows a named filter type to be in an imported file or module. Without this +// WithAllowIncludeOfImportedType returns an option for ImageFilteredByTypesWithOptions +// that allows a named included type to be in an imported file or module. Without this // option, only types defined directly in the image to be filtered are allowed. -func WithAllowFilterByImportedType() ImageFilterOption { +// Excluded types are always allowed to be in imported files or modules. +func WithAllowIncludeOfImportedType() ImageFilterOption { return func(opts *imageFilterOptions) { opts.allowImportedTypes = true } @@ -103,9 +104,10 @@ func WithAllowFilterByImportedType() ImageFilterOption { // the set of types that should be included in the filtered image. // // May be provided multiple times. The type names should be fully qualified. -// For example, "google.protobuf.Any" or "buf.validate". Types and package names -// are accepted. Types may be nested, and can be Messages, Enums, Services, Methods, -// or Extensions. If the type does not exist in the image, an error wrapping +// For example, "google.protobuf.Any" or "buf.validate". Types may be nested, +// and can be any package, message, enum, extension, service or method name. +// +// If the type does not exist in the image, an error wrapping // [ErrImageFilterTypeNotFound] will be returned. func WithIncludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { @@ -122,9 +124,10 @@ func WithIncludeTypes(typeNames ...string) ImageFilterOption { // specifies the set of types that should be excluded from the filtered image. // // May be provided multiple times. The type names should be fully qualified. -// For example, "google.protobuf.Any" or "buf.validate". Types and package names -// are accepted. Types may be nested, and can be Messages, Enums, Services, Methods, -// or Extensions. If the type does not exist in the image, an error wrapping +// For example, "google.protobuf.Any" or "buf.validate". Types may be nested, +// and can be any package, message, enum, extension, service or method name. +// +// If the type does not exist in the image, an error wrapping // [ErrImageFilterTypeNotFound] will be returned. func WithExcludeTypes(typeNames ...string) ImageFilterOption { return func(opts *imageFilterOptions) { @@ -560,19 +563,6 @@ func (t *transitiveClosure) excludeType( // but it's not... return fmt.Errorf("exclusion of type %q: %w", typeName, ErrImageFilterTypeNotFound) } - if !options.allowImportedTypes { - // if package includes only imported files, then reject - onlyImported := true - for _, file := range pkg.files { - if !file.IsImport() { - onlyImported = false - break - } - } - if onlyImported { - return fmt.Errorf("exclusion of type %q: %w", typeName, ErrImageFilterTypeIsImport) - } - } // Exclude the package and all of its files. for _, file := range pkg.files { fileDescriptor := file.FileDescriptorProto() diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index c4990e7596..8691e89521 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -362,7 +362,7 @@ func TestTypesFromMainModule(t *testing.T) { assert.ErrorIs(t, err, ErrImageFilterTypeIsImport) // allowed if we specify option - _, err = FilterImage(image, WithIncludeTypes("dependency.Dep"), WithAllowFilterByImportedType()) + _, err = FilterImage(image, WithIncludeTypes("dependency.Dep"), WithAllowIncludeOfImportedType()) require.NoError(t, err) _, err = FilterImage(image, WithIncludeTypes("nonexisting")) From 5decf0b03d5c816b7104d2673f5b91ceb3ed21fb Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 16:32:04 +0000 Subject: [PATCH 33/80] Add testcases --- .../cmd/buf/command/generate/generate_test.go | 23 +++++++++++++++++++ .../generate/testdata/v2/types/a/v1/a.proto | 23 +++++++++++++++++++ .../generate/testdata/v2/types/b/v1/b.proto | 9 ++++++++ .../testdata/v2/types/buf.gen.invalid.yaml | 11 +++++++++ .../generate/testdata/v2/types/buf.gen.yaml | 23 +++++++++++++++++++ .../generate/testdata/v2/types/buf.yaml | 1 + .../testdata/v2/types/pkg/v1/options.proto | 19 +++++++++++++++ .../bufimage/bufimageutil/bufimageutil.go | 11 +++++++-- .../bufimageutil/bufimageutil_test.go | 12 ++++++++++ 9 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto create mode 100644 private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto create mode 100644 private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.invalid.yaml create mode 100644 private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.yaml create mode 100644 private/buf/cmd/buf/command/generate/testdata/v2/types/buf.yaml create mode 100644 private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto diff --git a/private/buf/cmd/buf/command/generate/generate_test.go b/private/buf/cmd/buf/command/generate/generate_test.go index ea23236f0b..25ccb76bca 100644 --- a/private/buf/cmd/buf/command/generate/generate_test.go +++ b/private/buf/cmd/buf/command/generate/generate_test.go @@ -317,6 +317,29 @@ inputs: "--type", "b.v1.Bar", ) + // Include and exclude types on input and plugin. + testRunTypeArgs(t, map[string][]byte{ + filepath.Join("gen", "a", "v1", "a.top-level-type-names.yaml"): []byte(`messages: + - a.v1.Foo +`), + filepath.Join("gen", "b", "v1", "b.top-level-type-names.yaml"): []byte(`messages: + - b.v1.Baz +`), + }, + "--template", + filepath.Join("testdata", "v2", "types", "buf.gen.yaml"), + ) + // --exclude-type + testRunTypeArgs(t, map[string][]byte{ + filepath.Join("gen", "a", "v1", "a.top-level-type-names.yaml"): []byte(`messages: + - a.v1.Foo +`), + }, + "--template", + filepath.Join("testdata", "v2", "types", "buf.gen.yaml"), + "--exclude-type", + "b.v1.Baz", + ) } func TestOutputFlag(t *testing.T) { diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto b/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto new file mode 100644 index 0000000000..f0ac539061 --- /dev/null +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package a.v1; + +import "pkg/v1/options.proto"; +import "b/v1/b.proto"; + +message Foo { + option (pkg.v1.message_foo).foo = "str"; + message Bar { + option (pkg.v1.message_bar).bar = "str"; + string bar = 1; + } + Bar nested_bar = 1; + b.v1.Baz baz = 2; +} + +message Empty {} + +message FooBar { + Foo foo = 1; + Foo.Bar bar = 2; +} diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto b/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto new file mode 100644 index 0000000000..75a12c13cc --- /dev/null +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package b.v1; + +import "pkg/v1/options.proto"; + +message Baz { + option (pkg.v1.message_baz) = "str"; +} diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.invalid.yaml b/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.invalid.yaml new file mode 100644 index 0000000000..97827b3096 --- /dev/null +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.invalid.yaml @@ -0,0 +1,11 @@ +version: v2 +plugins: + - local: protoc-gen-top-level-type-names-yaml + out: gen + strategy: all +inputs: + - directory: ./testdata/v2/types + types: + - "a.v1.Foo.Bar" + exclude_types: + - "a.v1.Foo" diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.yaml b/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.yaml new file mode 100644 index 0000000000..76e3f7cdd8 --- /dev/null +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.gen.yaml @@ -0,0 +1,23 @@ +version: v2 +managed: + enabled: true + override: + - file_option: java_package_prefix + value: net +plugins: + - local: protoc-gen-top-level-type-names-yaml + out: gen + strategy: all + types: + - "a.v1.Foo" + exclude_types: + - "a.v1.Foo.Bar" + - "pkg.v1.message_foo" + - "pkg.v1.message_bar" +inputs: + - directory: ./testdata/v2/types + types: + - "a.v1.FooBar" + exclude_types: + - "a.v1.Empty" + - "pkg.v1.message_baz" diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.yaml b/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.yaml new file mode 100644 index 0000000000..b4d4e478b7 --- /dev/null +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/buf.yaml @@ -0,0 +1 @@ +version: v2 diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto b/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto new file mode 100644 index 0000000000..18c8301071 --- /dev/null +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +import "google/protobuf/descriptor.proto"; + +package pkg.v1; + +message OptionFoo { + string foo = 1; +} + +message OptionBar { + string bar = 1; +} + +extend google.protobuf.MessageOptions { + optional OptionFoo message_foo = 50000; + optional OptionBar message_bar = 50001; + optional string message_baz = 50002; +} diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 3a5c8f922a..6c1094106e 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -360,7 +360,14 @@ func (t *transitiveClosure) includeType( if !options.allowImportedTypes && descriptorInfo.file.IsImport() { return fmt.Errorf("inclusion of type %q: %w", typeName, ErrImageFilterTypeIsImport) } - return t.addElement(descriptorInfo.element, "", false, imageIndex, options) + // Check if the type is already excluded. + if mode := t.elements[descriptorInfo.element]; mode == inclusionModeExcluded { + return fmt.Errorf("inclusion of excluded type %q", typeName) + } + if err := t.addElement(descriptorInfo.element, "", false, imageIndex, options); err != nil { + return fmt.Errorf("inclusion of type %q: %w", typeName, err) + } + return nil } // It could be a package name pkg, ok := imageIndex.Packages[string(typeName)] @@ -384,7 +391,7 @@ func (t *transitiveClosure) includeType( for _, file := range pkg.files { fileDescriptor := file.FileDescriptorProto() if err := t.addElement(fileDescriptor, "", false, imageIndex, options); err != nil { - return err + return fmt.Errorf("inclusion of type %q: %w", typeName, err) } } return nil diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 8691e89521..e607b714e1 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -108,6 +108,7 @@ func TestTypes(t *testing.T) { t.Parallel() runDiffTest(t, "testdata/options", "pkg.FooService.mixed.txtar", WithIncludeTypes("pkg.FooService"), WithExcludeTypes("pkg.FooService.Do")) }) + } func TestNesting(t *testing.T) { @@ -150,6 +151,17 @@ func TestNesting(t *testing.T) { t.Parallel() runDiffTest(t, "testdata/nesting", "mixed.txtar", WithIncludeTypes("pkg.Foo", "pkg.FooEnum"), WithExcludeTypes("pkg.Foo.NestedFoo", "pkg.Baz")) }) + + t.Run("include-excluded", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/nesting", bufimage.WithExcludeSourceCodeInfo()) + require.NoError(t, err) + _, err = FilterImage(image, WithIncludeTypes("pkg.Foo.NestedFoo"), WithExcludeTypes("pkg.Foo")) + require.ErrorContains(t, err, "inclusion of excluded type \"pkg.Foo.NestedFoo\"") + _, err = FilterImage(image, WithIncludeTypes("pkg.Foo.NestedButNotUsed"), WithExcludeTypes("pkg.Foo")) + require.ErrorContains(t, err, "inclusion of excluded type \"pkg.Foo.NestedButNotUsed\"") + }) } func TestOptions(t *testing.T) { From e07f2b7471b86edc084863a3788a8a8deedb28f4 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 17:36:34 +0000 Subject: [PATCH 34/80] Fix lint --- private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index e607b714e1..b2a9fe6944 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -108,7 +108,6 @@ func TestTypes(t *testing.T) { t.Parallel() runDiffTest(t, "testdata/options", "pkg.FooService.mixed.txtar", WithIncludeTypes("pkg.FooService"), WithExcludeTypes("pkg.FooService.Do")) }) - } func TestNesting(t *testing.T) { From a3f2055f3ad6019230519ce21be5f0092ef388ec Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 22:13:20 +0000 Subject: [PATCH 35/80] Cleanup --- private/buf/bufctl/controller.go | 4 +--- private/buf/bufctl/option.go | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 631ec8762f..a76ad26144 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -1330,9 +1330,7 @@ func filterImage( } includeTypes := functionOptions.imageTypes excludeTypes := functionOptions.imageExcludeTypes - includeOptions := functionOptions.imageOptions - excludeOptions := functionOptions.imageExcludeOptions - if len(includeTypes) > 0 || len(excludeTypes) > 0 || len(includeOptions) > 0 || len(excludeOptions) > 0 { + if len(includeTypes) > 0 || len(excludeTypes) > 0 { newImage, err = bufimageutil.FilterImage( newImage, bufimageutil.WithIncludeTypes(includeTypes...), diff --git a/private/buf/bufctl/option.go b/private/buf/bufctl/option.go index a0f07ddbd8..5a0d2d419b 100644 --- a/private/buf/bufctl/option.go +++ b/private/buf/bufctl/option.go @@ -141,8 +141,6 @@ type functionOptions struct { imageExcludeImports bool imageTypes []string imageExcludeTypes []string - imageOptions []string - imageExcludeOptions []string imageAsFileDescriptorSet bool configOverride string ignoreAndDisallowV1BufWorkYAMLs bool From b8ed6d7b3f79303752e1325bb76ca53635ab9e5f Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 22:15:00 +0000 Subject: [PATCH 36/80] Cleanup --- private/buf/bufgen/generator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 589c4ed610..b3bb77e0ee 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -213,7 +213,6 @@ func (g *generator) execPlugins( requiredFeatures := computeRequiredFeatures(image) // Group the pluginConfigs by the properties ExcludeOptions, Strategy, and RemoteHost. - pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, createPluginConfigKeyForImage) if err != nil { return nil, err From cbfb29f05fb63a90811b09898f05542204e261df Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 11 Mar 2025 22:18:52 +0000 Subject: [PATCH 37/80] Cleanup --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 6c1094106e..63f7683a72 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -806,11 +806,9 @@ func (t *transitiveClosure) exploreCustomOptions( return fmt.Errorf("unexpected type for exploring options %T", descriptor) } - count := 0 optionsName := options.Descriptor().FullName() var err error options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - count++ if !t.hasOption(fd, imageIndex, opts) { return true } From 13c3882108cf8e9bb10ea16c918a843b8e17ae1b Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 14 Mar 2025 22:09:28 +0000 Subject: [PATCH 38/80] Use slices.Backward --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index b790ebefea..66cfc3478a 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -69,8 +69,7 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im dirty := false newImageFiles := make([]bufimage.ImageFile, 0, len(image.Files())) importsByFilePath := make(map[string]struct{}) - for i := len(image.Files()) - 1; i >= 0; i-- { - imageFile := imageFiles[i] + for _, imageFile := range slices.Backward(imageFiles) { imageFilePath := imageFile.Path() _, isFileImported := importsByFilePath[imageFilePath] if imageFile.IsImport() && !options.allowImportedTypes { From 3124c00674a3e262864f40cac090fb2b21833524 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 14 Mar 2025 22:09:41 +0000 Subject: [PATCH 39/80] Improve generator key --- private/buf/bufgen/generator.go | 46 ++++++++++++++------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index b3bb77e0ee..60530f8430 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -16,7 +16,6 @@ package bufgen import ( "context" - "encoding/json" "errors" "fmt" "log/slog" @@ -213,10 +212,7 @@ func (g *generator) execPlugins( requiredFeatures := computeRequiredFeatures(image) // Group the pluginConfigs by the properties ExcludeOptions, Strategy, and RemoteHost. - pluginConfigsForImage, err := slicesext.ToIndexedValuesMapError(pluginConfigs, createPluginConfigKeyForImage) - if err != nil { - return nil, err - } + pluginConfigsForImage := slicesext.ToIndexedValuesMap(pluginConfigs, createPluginConfigKeyForImage) for _, indexedPluginConfigs := range pluginConfigsForImage { image := image hashPluginConfig := indexedPluginConfigs[0].Value @@ -494,30 +490,28 @@ func newGenerateOptions() *generateOptions { return &generateOptions{} } -// createPluginConfigKeyForImage returns a string of the plugin config with +type pluginConfigKeyForImage struct { + types string // string representation of []string + excludeTypes string // string representation of []string + strategy Strategy + remoteHost string +} + +// createPluginConfigKeyForImage returns a key of the plugin config with // a subset of properties. This is used to batch plugins that have similar // configuration. The key is based on the following properties: -// - ExcludeOptions +// - Types +// - ExcludeTypes // - Strategy // - RemoteHost -func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) (string, error) { - type pluginConfigKey struct { - Types []string `json:"types"` - ExcludeTypes []string `json:"exclude_types"` - Strategy Strategy `json:"strategy"` - RemoteHost string `json:"remote_host"` - } - key := &pluginConfigKey{ - Types: pluginConfig.Types(), - ExcludeTypes: pluginConfig.ExcludeTypes(), - Strategy: Strategy(pluginConfig.Strategy()), - RemoteHost: pluginConfig.RemoteHost(), - } - sort.Strings(key.Types) - sort.Strings(key.ExcludeTypes) - bytes, err := json.Marshal(key) - if err != nil { - return "", err +func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) pluginConfigKeyForImage { + // Sort the types and excludeTypes so that the key is deterministic. + sort.Strings(pluginConfig.Types()) + sort.Strings(pluginConfig.ExcludeTypes()) + return pluginConfigKeyForImage{ + types: fmt.Sprintf("%v", pluginConfig.Types()), + excludeTypes: fmt.Sprintf("%v", pluginConfig.ExcludeTypes()), + strategy: Strategy(pluginConfig.Strategy()), + remoteHost: pluginConfig.RemoteHost(), } - return string(bytes), nil } From eaf49d280d65d85c6d06b568a0541b4369f976ca Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 17 Mar 2025 23:56:22 +0100 Subject: [PATCH 40/80] Fix unused deps --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 66cfc3478a..65d0dd347d 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -163,7 +163,7 @@ func filterImageFile( imageFile.LocalPath(), imageFile.IsImport(), imageFile.IsSyntaxUnspecified(), - imageFile.UnusedDependencyIndexes(), + nil, // There are no unused dependencies. ) } From 59f5d786349f7805b5ecea055d8069821befa115 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 18 Mar 2025 00:19:57 +0100 Subject: [PATCH 41/80] Add unused deps testcase --- .../bufimageutil/bufimageutil_test.go | 24 +++++++++++++++---- .../bufimageutil/testdata/unuseddeps/a.proto | 7 ++++++ .../bufimageutil/testdata/unuseddeps/a.txtar | 12 ++++++++++ .../bufimageutil/testdata/unuseddeps/ab.txtar | 5 ++++ .../bufimageutil/testdata/unuseddeps/b.proto | 3 +++ .../bufimageutil/testdata/unuseddeps/c.proto | 3 +++ 6 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/ab.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/b.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/c.proto diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index b2a9fe6944..151af55139 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -28,6 +28,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/protoencoding" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storageos" @@ -309,6 +310,12 @@ func TestSourceCodeInfo(t *testing.T) { runSourceCodeInfoTest(t, "foo.bar", "all.txtar") } +func TestUnusedDeps(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/unuseddeps", "a.txtar", WithIncludeTypes("a.A")) + runDiffTest(t, "testdata/unuseddeps", "ab.txtar", WithIncludeTypes("a.A"), WithExcludeTypes("b.B")) +} + func TestTransitivePublic(t *testing.T) { t.Parallel() ctx := context.Background() @@ -451,15 +458,22 @@ func runFilterImage(t *testing.T, image bufimage.Image, opts ...ImageFilterOptio assert.NotNil(t, image) assert.True(t, imageIsDependencyOrdered(filteredImage), "image files not in dependency order") + // Convert the filtered image back to a proto image and then back to an image to ensure that the + // image is still valid after filtering. + protoImage, err := bufimage.ImageToProtoImage(filteredImage) + require.NoError(t, err) + filteredImage, err = bufimage.NewImageForProto(protoImage) + require.NoError(t, err) + // We may have filtered out custom options from the set in the step above. However, the options messages // still contain extension fields that refer to the custom options, as a result of building the image. // So we serialize and then de-serialize, and use only the filtered results to parse extensions. That way, the result will omit custom options that aren't present in the filtered set (as they will be // considered unrecognized fields). - data, err := protoencoding.NewWireMarshaler().Marshal(bufimage.ImageToFileDescriptorSet(filteredImage)) - require.NoError(t, err) - fileDescriptorSet := &descriptorpb.FileDescriptorSet{} - err = protoencoding.NewWireUnmarshaler(filteredImage.Resolver()).Unmarshal(data, fileDescriptorSet) - require.NoError(t, err) + fileDescriptorSet := &descriptorpb.FileDescriptorSet{ + File: slicesext.Map(filteredImage.Files(), func(imageFile bufimage.ImageFile) *descriptorpb.FileDescriptorProto { + return imageFile.FileDescriptorProto() + }), + } files, err := protodesc.NewFiles(fileDescriptorSet) require.NoError(t, err) diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.proto new file mode 100644 index 0000000000..79fde64ceb --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package a; +import "b.proto"; +import "c.proto"; +message A{ + b.B b = 1; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.txtar new file mode 100644 index 0000000000..8d9134f706 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/a.txtar @@ -0,0 +1,12 @@ +-- a.proto -- +syntax = "proto3"; +package a; +import "b.proto"; +message A { + b.B b = 1; +} +-- b.proto -- +syntax = "proto3"; +package b; +message B { +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/ab.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/ab.txtar new file mode 100644 index 0000000000..afce27cb76 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/ab.txtar @@ -0,0 +1,5 @@ +-- a.proto -- +syntax = "proto3"; +package a; +message A { +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/b.proto b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/b.proto new file mode 100644 index 0000000000..1741b295bb --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/b.proto @@ -0,0 +1,3 @@ +syntax = "proto3"; +package b; +message B {} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/c.proto b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/c.proto new file mode 100644 index 0000000000..2dcba3375b --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/unuseddeps/c.proto @@ -0,0 +1,3 @@ +syntax = "proto3"; +package c; +message C {} From 07152c5d3294012bfbc50e8b80b6ca758c6492af Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 18 Mar 2025 00:20:39 +0100 Subject: [PATCH 42/80] Fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 957f6feb96..79a1c3d0aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [Unreleased] - Fix `buf convert` to allow for zero length for `binpb`, `txtpb`, and `yaml` formats. -- Improved type filtering for `buf generate`. Adds the ability to exclude types with the parameter +- Improve type filtering for `buf generate`. Adds the ability to exclude types with the parameter `exclude_types` in `buf.gen.yaml` and a flag `--exclude-types` in the CLI. Type filters may now also be specified as plugin parameters in `buf.gen.yaml`. From cd436001beb4aadffd520e25dc62613c4a945e0c Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 18 Mar 2025 11:53:55 +0100 Subject: [PATCH 43/80] Fix mutate in place source code locations --- .../bufimageutil/bufimageutil_test.go | 10 ++++++-- .../bufimage/bufimageutil/image_filter.go | 25 +++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 151af55139..3d1488ed45 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -391,7 +391,7 @@ func TestTypesFromMainModule(t *testing.T) { func TestMutateInPlace(t *testing.T) { t.Parallel() ctx := context.Background() - _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/options", bufimage.WithExcludeSourceCodeInfo()) + _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/options") require.NoError(t, err) aProtoFile := image.GetFile("a.proto") @@ -400,11 +400,16 @@ func TestMutateInPlace(t *testing.T) { assert.Len(t, aFileDescriptorProto.EnumType, 1) // FooEnum assert.Len(t, aFileDescriptorProto.Service, 1) + locationLen := len(aFileDescriptorProto.SourceCodeInfo.Location) + // Shallow copy shallowFilteredImage, err := FilterImage(image, WithIncludeTypes("pkg.Foo")) require.NoError(t, err) - assert.NotSame(t, aFileDescriptorProto, shallowFilteredImage.GetFile("a.proto").FileDescriptorProto()) + filteredAFileDescriptorProto := shallowFilteredImage.GetFile("a.proto").FileDescriptorProto() + assert.NotSame(t, aFileDescriptorProto, filteredAFileDescriptorProto) + filterLocationLen := len(filteredAFileDescriptorProto.SourceCodeInfo.Location) + assert.Less(t, filterLocationLen, locationLen) // Mutate in place mutateFilteredImage, err := FilterImage(image, WithIncludeTypes("pkg.Foo"), WithMutateInPlace()) @@ -421,6 +426,7 @@ func TestMutateInPlace(t *testing.T) { } assert.Nil(t, aFileDescriptorProto.EnumType) assert.Nil(t, aFileDescriptorProto.Service) + assert.Equal(t, filterLocationLen, len(aFileDescriptorProto.SourceCodeInfo.Location)) } func getImage(ctx context.Context, logger *slog.Logger, testdataDir string, options ...bufimage.BuildImageOption) (storage.ReadWriteBucket, bufimage.Image, error) { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 65d0dd347d..86feebf7c6 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -120,14 +120,14 @@ func filterImageFile( closure: closure, options: options, } - newFileDescriptor, err := builder.remapFileDescriptor(&sourcePathsRemap, fileDescriptor) + newFileDescriptor, changed, err := builder.remapFileDescriptor(&sourcePathsRemap, fileDescriptor) if err != nil { return nil, err } if newFileDescriptor == nil { return nil, nil // Filtered out. } - if newFileDescriptor == fileDescriptor { + if !changed { return imageFile, nil // No changes required. } @@ -167,6 +167,9 @@ func filterImageFile( ) } +// sourcePathsBuilder is a helper for building the new source paths. +// Each method return the new value, whether it was changed, and an error if any. +// The value is nil if it was filtered out. type sourcePathsBuilder struct { imageIndex *imageIndex closure *transitiveClosure @@ -176,9 +179,9 @@ type sourcePathsBuilder struct { func (b *sourcePathsBuilder) remapFileDescriptor( sourcePathsRemap *sourcePathsRemapTrie, fileDescriptor *descriptorpb.FileDescriptorProto, -) (*descriptorpb.FileDescriptorProto, error) { +) (*descriptorpb.FileDescriptorProto, bool, error) { if !b.closure.hasType(fileDescriptor, b.options) { - return nil, nil + return nil, true, nil } sourcePath := make(protoreflect.SourcePath, 0, 8) @@ -187,32 +190,32 @@ func (b *sourcePathsBuilder) remapFileDescriptor( isDirty := false newMessages, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileMessagesTag), fileDescriptor.MessageType, b.remapDescriptor, b.options) if err != nil { - return nil, err + return nil, false, err } isDirty = isDirty || changed newEnums, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileEnumsTag), fileDescriptor.EnumType, b.remapEnum, b.options) if err != nil { - return nil, err + return nil, false, err } isDirty = isDirty || changed newServices, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileServicesTag), fileDescriptor.Service, b.remapService, b.options) if err != nil { - return nil, err + return nil, false, err } isDirty = isDirty || changed newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, fileExtensionsTag), fileDescriptor.Extension, b.remapField, b.options) if err != nil { - return nil, err + return nil, false, err } isDirty = isDirty || changed newOptions, changed, err := remapMessage(nil, append(sourcePath, fileOptionsTag), fileDescriptor.Options, b.remapOptions) if err != nil { - return nil, err + return nil, false, err } isDirty = isDirty || changed if !isDirty { - return fileDescriptor, nil + return fileDescriptor, false, nil } newFileDescriptor := maybeClone(fileDescriptor, b.options) @@ -270,7 +273,7 @@ func (b *sourcePathsBuilder) remapFileDescriptor( } } } - return newFileDescriptor, nil + return newFileDescriptor, true, nil } func (b *sourcePathsBuilder) remapDescriptor( From f291c2aecbb800f32d86a47817e2109be1f79a94 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 18 Mar 2025 12:06:30 +0100 Subject: [PATCH 44/80] Update docs --- private/buf/bufgen/generator.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 60530f8430..0c9d43eb02 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -211,13 +211,13 @@ func (g *generator) execPlugins( responses := make([]*pluginpb.CodeGeneratorResponse, len(pluginConfigs)) requiredFeatures := computeRequiredFeatures(image) - // Group the pluginConfigs by the properties ExcludeOptions, Strategy, and RemoteHost. + // Group the pluginConfigs by similar properties to batch image processing. pluginConfigsForImage := slicesext.ToIndexedValuesMap(pluginConfigs, createPluginConfigKeyForImage) for _, indexedPluginConfigs := range pluginConfigsForImage { image := image hashPluginConfig := indexedPluginConfigs[0].Value - // Apply per-config filters. + // Apply per-plugin filters. includeTypes := hashPluginConfig.Types() excludeTypes := hashPluginConfig.ExcludeTypes() if len(includeTypes) > 0 || len(excludeTypes) > 0 { @@ -491,7 +491,7 @@ func newGenerateOptions() *generateOptions { } type pluginConfigKeyForImage struct { - types string // string representation of []string + includeTypes string // string representation of []string excludeTypes string // string representation of []string strategy Strategy remoteHost string @@ -509,7 +509,7 @@ func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) sort.Strings(pluginConfig.Types()) sort.Strings(pluginConfig.ExcludeTypes()) return pluginConfigKeyForImage{ - types: fmt.Sprintf("%v", pluginConfig.Types()), + includeTypes: fmt.Sprintf("%v", pluginConfig.Types()), excludeTypes: fmt.Sprintf("%v", pluginConfig.ExcludeTypes()), strategy: Strategy(pluginConfig.Strategy()), remoteHost: pluginConfig.RemoteHost(), From e78e983144d90aecad8737a5595ee7a4c8aeafb5 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 17:27:38 +0100 Subject: [PATCH 45/80] Fix name pluginConfigForKey --- private/buf/bufgen/generator.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 0c9d43eb02..685bae8959 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -215,11 +215,11 @@ func (g *generator) execPlugins( pluginConfigsForImage := slicesext.ToIndexedValuesMap(pluginConfigs, createPluginConfigKeyForImage) for _, indexedPluginConfigs := range pluginConfigsForImage { image := image - hashPluginConfig := indexedPluginConfigs[0].Value + pluginConfigForKey := indexedPluginConfigs[0].Value // Apply per-plugin filters. - includeTypes := hashPluginConfig.Types() - excludeTypes := hashPluginConfig.ExcludeTypes() + includeTypes := pluginConfigForKey.Types() + excludeTypes := pluginConfigForKey.ExcludeTypes() if len(includeTypes) > 0 || len(excludeTypes) > 0 { var err error image, err = bufimageutil.FilterImage( @@ -233,7 +233,7 @@ func (g *generator) execPlugins( } // Batch for each remote. - if remote := hashPluginConfig.RemoteHost(); remote != "" { + if remote := pluginConfigForKey.RemoteHost(); remote != "" { jobs = append(jobs, func(ctx context.Context) error { results, err := g.execRemotePluginsV2( ctx, @@ -257,7 +257,7 @@ func (g *generator) execPlugins( // Local plugins. var images []bufimage.Image - switch Strategy(hashPluginConfig.Strategy()) { + switch Strategy(pluginConfigForKey.Strategy()) { case StrategyAll: images = []bufimage.Image{image} case StrategyDirectory: @@ -267,15 +267,15 @@ func (g *generator) execPlugins( return nil, err } default: - return nil, fmt.Errorf("unknown strategy: %v", hashPluginConfig.Strategy()) + return nil, fmt.Errorf("unknown strategy: %v", pluginConfigForKey.Strategy()) } for _, indexedPluginConfig := range indexedPluginConfigs { jobs = append(jobs, func(ctx context.Context) error { - includeImports := hashPluginConfig.IncludeImports() + includeImports := pluginConfigForKey.IncludeImports() if includeImportsOverride != nil { includeImports = *includeImportsOverride } - includeWellKnownTypes := hashPluginConfig.IncludeWKT() + includeWellKnownTypes := pluginConfigForKey.IncludeWKT() if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } From e39dddd3866cb70a470ef68e63b7471e91629fe4 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 17:28:57 +0100 Subject: [PATCH 46/80] Fix pluginConfig ref for local job --- private/buf/bufgen/generator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 685bae8959..f9294d931f 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -271,11 +271,11 @@ func (g *generator) execPlugins( } for _, indexedPluginConfig := range indexedPluginConfigs { jobs = append(jobs, func(ctx context.Context) error { - includeImports := pluginConfigForKey.IncludeImports() + includeImports := indexedPluginConfig.Value.IncludeImports() if includeImportsOverride != nil { includeImports = *includeImportsOverride } - includeWellKnownTypes := pluginConfigForKey.IncludeWKT() + includeWellKnownTypes := indexedPluginConfig.Value.IncludeWKT() if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } From 56c5fdaa6bad1d33ab4d6a54ef622b25f5b85b42 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 17:30:21 +0100 Subject: [PATCH 47/80] Fix indexedPluginConfig consistency --- private/buf/bufgen/generator.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index f9294d931f..ce49404fd0 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -365,17 +365,17 @@ func (g *generator) execRemotePluginsV2( includeWellKnownTypesOverride *bool, ) ([]slicesext.Indexed[*pluginpb.CodeGeneratorResponse], error) { requests := make([]*registryv1alpha1.PluginGenerationRequest, len(indexedPluginConfigs)) - for i, pluginConfig := range indexedPluginConfigs { - includeImports := pluginConfig.Value.IncludeImports() + for i, indexedPluginConfig := range indexedPluginConfigs { + includeImports := indexedPluginConfig.Value.IncludeImports() if includeImportsOverride != nil { includeImports = *includeImportsOverride } - includeWellKnownTypes := pluginConfig.Value.IncludeWKT() + includeWellKnownTypes := indexedPluginConfig.Value.IncludeWKT() if includeWellKnownTypesOverride != nil { includeWellKnownTypes = *includeWellKnownTypesOverride } request, err := getPluginGenerationRequest( - pluginConfig.Value, + indexedPluginConfig.Value, includeImports, includeWellKnownTypes, ) From 9a80424cae5ffa47eaeacc88a18c0f69debc76cc Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 17:55:04 +0100 Subject: [PATCH 48/80] Update --exclude-types comment --- private/buf/cmd/buf/command/generate/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 190d6b184b..6de8164045 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -482,7 +482,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { &f.ExcludeTypes, excludeTypeFlagName, nil, - "The types (package, message, enum, extension, service, method) that should be excluded from this image. When specified, the resulting image will not include descriptors to describe the requested types. Flag usage overrides buf.gen.yaml", + "The types (package, message, enum, extension, service, method) that should be excluded from this image. When specified, the resulting image will omit descriptors for the specified types and remove any references to them, such as fields typed to an excluded message or enum, or custom options tied to an excluded extension. The image is first filtered by the included types, then further reduced by the excluded. Flag usage overrides buf.gen.yaml", ) _ = flagSet.MarkDeprecated(typeDeprecatedFlagName, fmt.Sprintf("use --%s instead", typeFlagName)) _ = flagSet.MarkHidden(typeDeprecatedFlagName) From 60dbf41a635e75e04d4994f3d7cda3dd09dc7886 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 17:56:46 +0100 Subject: [PATCH 49/80] Remove dead comment --- private/buf/cmd/buf/command/generate/generate.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 6de8164045..8c9163e74c 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -652,8 +652,6 @@ func getInputImages( if len(excludePathsOverride) > 0 { excludePaths = excludePathsOverride } - // In V1 we do not need to look at generateTypeConfig.IncludeTypes() - // because it is always nil. includeTypes := inputConfig.IncludeTypes() if len(includeTypesOverride) > 0 { includeTypes = includeTypesOverride From a57873e3760432f290528d0131ecd714b97cfb0a Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 17:59:32 +0100 Subject: [PATCH 50/80] Clarify test comment --- private/buf/cmd/buf/command/generate/generate_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/private/buf/cmd/buf/command/generate/generate_test.go b/private/buf/cmd/buf/command/generate/generate_test.go index 25ccb76bca..f70fc98c85 100644 --- a/private/buf/cmd/buf/command/generate/generate_test.go +++ b/private/buf/cmd/buf/command/generate/generate_test.go @@ -317,7 +317,7 @@ inputs: "--type", "b.v1.Bar", ) - // Include and exclude types on input and plugin. + // buf.gen.yaml has types and exclude_types on inputs and plugins testRunTypeArgs(t, map[string][]byte{ filepath.Join("gen", "a", "v1", "a.top-level-type-names.yaml"): []byte(`messages: - a.v1.Foo @@ -329,7 +329,7 @@ inputs: "--template", filepath.Join("testdata", "v2", "types", "buf.gen.yaml"), ) - // --exclude-type + // --exclude-type override testRunTypeArgs(t, map[string][]byte{ filepath.Join("gen", "a", "v1", "a.top-level-type-names.yaml"): []byte(`messages: - a.v1.Foo From 986a33fcdad571396b53dc8cd9dadacc039e7bed Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:01:40 +0100 Subject: [PATCH 51/80] Use IncludeTypes --- private/buf/bufgen/generator.go | 6 +++--- .../bufpkg/bufconfig/generate_plugin_config.go | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index ce49404fd0..d5ac74d99f 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -218,7 +218,7 @@ func (g *generator) execPlugins( pluginConfigForKey := indexedPluginConfigs[0].Value // Apply per-plugin filters. - includeTypes := pluginConfigForKey.Types() + includeTypes := pluginConfigForKey.IncludeTypes() excludeTypes := pluginConfigForKey.ExcludeTypes() if len(includeTypes) > 0 || len(excludeTypes) > 0 { var err error @@ -506,10 +506,10 @@ type pluginConfigKeyForImage struct { // - RemoteHost func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) pluginConfigKeyForImage { // Sort the types and excludeTypes so that the key is deterministic. - sort.Strings(pluginConfig.Types()) + sort.Strings(pluginConfig.IncludeTypes()) sort.Strings(pluginConfig.ExcludeTypes()) return pluginConfigKeyForImage{ - includeTypes: fmt.Sprintf("%v", pluginConfig.Types()), + includeTypes: fmt.Sprintf("%v", pluginConfig.IncludeTypes()), excludeTypes: fmt.Sprintf("%v", pluginConfig.ExcludeTypes()), strategy: Strategy(pluginConfig.Strategy()), remoteHost: pluginConfig.RemoteHost(), diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index f6c39a947c..13cd92327b 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -112,8 +112,8 @@ type GeneratePluginConfig interface { // // This is not empty only when the plugin is remote. Revision() int - // Types returns the types to include. - Types() []string + // IncludeTypes returns the types to include. + IncludeTypes() []string // ExcludeTypes returns the types to exclude. ExcludeTypes() []string @@ -247,7 +247,7 @@ type generatePluginConfig struct { opts []string includeImports bool includeWKT bool - types []string + includeTypes []string excludeTypes []string strategy *GenerateStrategy path []string @@ -532,7 +532,7 @@ func newRemoteGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, - types: types, + includeTypes: types, excludeTypes: excludeTypes, }, nil } @@ -558,7 +558,7 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, - types: types, + includeTypes: types, excludeTypes: excludeTypes, }, nil } @@ -589,7 +589,7 @@ func newLocalGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, - types: types, + includeTypes: types, excludeTypes: excludeTypes, }, nil } @@ -617,7 +617,7 @@ func newProtocBuiltinGeneratePluginConfig( strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, - types: types, + includeTypes: types, excludeTypes: excludeTypes, }, nil } @@ -646,8 +646,8 @@ func (p *generatePluginConfig) IncludeWKT() bool { return p.includeWKT } -func (p *generatePluginConfig) Types() []string { - return p.types +func (p *generatePluginConfig) IncludeTypes() []string { + return p.includeTypes } func (p *generatePluginConfig) ExcludeTypes() []string { From e7de3e866110e69d301a30542328a2d64d421e71 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:06:15 +0100 Subject: [PATCH 52/80] Clarify input config ExcludeType docs --- private/bufpkg/bufconfig/input_config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 62e8bca7df..4294195334 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -161,10 +161,8 @@ type InputConfig interface { // ExcludePaths returns paths not to generate for. ExcludePaths() []string // IncludeTypes returns the types to generate. An empty slice means to generate for all types. - // This is unioned with ExcludeTypes. IncludeTypes() []string // ExcludeTypes returns the types to exclude. An empty slice means to exclude no types. - // This is unioned with IncludeTypes. ExcludeTypes() []string isInputConfig() From d7849e1931946113fcbec931e2dffa2f5578bb3f Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:10:08 +0100 Subject: [PATCH 53/80] Remove options == nil checks --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 6 ------ private/bufpkg/bufimage/bufimageutil/image_filter.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 63f7683a72..7768fa0b2a 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -305,9 +305,6 @@ func (t *transitiveClosure) hasType( descriptor namedDescriptor, options *imageFilterOptions, ) (isIncluded bool) { - if options == nil { - return true // no filter - } switch mode := t.elements[descriptor]; mode { case inclusionModeExplicit, inclusionModeImplicit, inclusionModeEnclosing: return true @@ -326,9 +323,6 @@ func (t *transitiveClosure) hasOption( imageIndex *imageIndex, options *imageFilterOptions, ) (isIncluded bool) { - if options == nil { - return true // no filter - } if !fieldDescriptor.IsExtension() { return true } diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 86feebf7c6..bbf4ea0b60 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -654,7 +654,7 @@ func remapSlice[T any]( } func maybeClone[T proto.Message](value T, options *imageFilterOptions) T { - if options == nil || !options.mutateInPlace { + if !options.mutateInPlace { return shallowClone(value) } return value From 62dc5d4cf1f5b3ed939e8bdc8e84013fb7086f5e Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:14:14 +0100 Subject: [PATCH 54/80] Move exclude set --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 7768fa0b2a..fee6271f29 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -586,7 +586,7 @@ func (t *transitiveClosure) excludeElement( } return nil } - + t.elements[descriptor] = inclusionModeExcluded switch descriptor := descriptor.(type) { case *descriptorpb.FileDescriptorProto: for _, descriptor := range descriptor.GetMessageType() { @@ -643,7 +643,6 @@ func (t *transitiveClosure) excludeElement( default: return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) } - t.elements[descriptor] = inclusionModeExcluded return nil } From 4c75c24827f82055d81d39590a09767e418bb27d Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:16:29 +0100 Subject: [PATCH 55/80] Move TODO for globs --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 2 -- private/bufpkg/bufimage/bufimageutil/image_filter.go | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index fee6271f29..aa642989da 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -346,8 +346,6 @@ func (t *transitiveClosure) includeType( imageIndex *imageIndex, options *imageFilterOptions, ) error { - // TODO: consider supporting a glob syntax of some kind, to do more advanced pattern - // matching, such as ability to get a package AND all of its sub-packages. descriptorInfo, ok := imageIndex.ByName[typeName] if ok { // It's a type name diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index bbf4ea0b60..ab777e6ad7 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -33,6 +33,8 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im } closure := newTransitiveClosure() // All excludes are added first, then includes walk included all non excluded types. + // TODO: consider supporting a glob syntax of some kind, to do more advanced pattern + // matching, such as ability to get a package AND all of its sub-packages. for excludeType := range options.excludeTypes { excludeType := protoreflect.FullName(excludeType) if err := closure.excludeType(excludeType, imageIndex, options); err != nil { From cd9d8a1f59d26e3a195273978f2632964d72924c Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:28:44 +0100 Subject: [PATCH 56/80] Improve excludeElement extendee check --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index aa642989da..fdca12753d 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -634,10 +634,13 @@ func (t *transitiveClosure) excludeElement( } case *descriptorpb.MethodDescriptorProto: case *descriptorpb.FieldDescriptorProto: - if descriptor.GetExtendee() == "" { + // Only extension fields can be excluded. + if descriptor.Extendee == nil { return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) } - // Is an extension field. + if descriptor.GetExtendee() == "" { + return fmt.Errorf("expected extendee for field %q to not be empty", descriptorInfo.fullName) + } default: return errorUnsupportedFilterType(descriptor, descriptorInfo.fullName) } From 219ff4b8f3b64f3e5badde81980b24d662d49b42 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 18:43:27 +0100 Subject: [PATCH 57/80] Clarify hasOption on unknown types --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index fdca12753d..e2cd938cda 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -332,10 +332,14 @@ func (t *transitiveClosure) hasOption( fullName := fieldDescriptor.FullName() descriptor := imageIndex.ByName[fullName].element switch mode := t.elements[descriptor]; mode { - case inclusionModeExplicit, inclusionModeImplicit, inclusionModeEnclosing, inclusionModeUnknown: + case inclusionModeExplicit, inclusionModeImplicit, inclusionModeEnclosing: return true case inclusionModeExcluded: return false + case inclusionModeUnknown: + // True as option type is not explicitly excluded. + // Occurs on first transversal when adding included types. + return true default: return false } From c20a21c93d0f9f74272431b8b8cd7a25a30a833f Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 19:01:55 +0100 Subject: [PATCH 58/80] Doc check is import used --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index ab777e6ad7..12bfd820aa 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -74,11 +74,10 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im for _, imageFile := range slices.Backward(imageFiles) { imageFilePath := imageFile.Path() _, isFileImported := importsByFilePath[imageFilePath] - if imageFile.IsImport() && !options.allowImportedTypes { - // Check if this import is still used. - if !isFileImported { - continue - } + // Check if this import is still used. If allowImportedTypes is true, we + // must check the imported file to see if it is used. + if imageFile.IsImport() && !options.allowImportedTypes && !isFileImported { + continue } newImageFile, err := filterImageFile( imageFile, From 9e8f745a894aba491bd8c2558130b4509064bbd2 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 19:07:37 +0100 Subject: [PATCH 59/80] TODO for better enclosing isDirty check --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 12bfd820aa..4640ac84da 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -100,12 +100,12 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im } newImageFiles = append(newImageFiles, newImageFile) } - if dirty { - // Reverse the image files back to DAG order. - slices.Reverse(newImageFiles) - return bufimage.NewImage(newImageFiles) + if !dirty { + return image, nil } - return image, nil + // Reverse the image files back to DAG order. + slices.Reverse(newImageFiles) + return bufimage.NewImage(newImageFiles) } func filterImageFile( @@ -289,6 +289,8 @@ func (b *sourcePathsBuilder) remapDescriptor( isDirty := false if mode := b.closure.elements[descriptor]; mode == inclusionModeEnclosing { // If the type is only enclosing, only the namespace matters. + // TODO: improve isDirty check to avoid cloning the descriptor + // if this type is already only an enclosing type. isDirty = true newDescriptor = maybeClone(descriptor, b.options) sourcePathsRemap.markNoComment(sourcePath) From bc52487ccf0c1b0a334ad4dd7a0970816851f7e3 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 19:21:26 +0100 Subject: [PATCH 60/80] Clarify input config docs for ExcludeTypes --- private/bufpkg/bufconfig/input_config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 4294195334..acf01c9419 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -163,6 +163,10 @@ type InputConfig interface { // IncludeTypes returns the types to generate. An empty slice means to generate for all types. IncludeTypes() []string // ExcludeTypes returns the types to exclude. An empty slice means to exclude no types. + // First ExcludeTypes are removed from the image. + // Then IncludeTypes are traversed to compute the full set of types in the schema. + // Not only are the types removed, but all references to the types must also be + // removed in order for the resulting schema to be valid. ExcludeTypes() []string isInputConfig() From 7b12e9800aa79acb2741064746cf25d2d8dd6ff3 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 20:57:16 +0100 Subject: [PATCH 61/80] Do TODO for enclosingType --- .../bufimage/bufimageutil/image_filter.go | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 4640ac84da..59fa1df4fa 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -289,21 +289,26 @@ func (b *sourcePathsBuilder) remapDescriptor( isDirty := false if mode := b.closure.elements[descriptor]; mode == inclusionModeEnclosing { // If the type is only enclosing, only the namespace matters. - // TODO: improve isDirty check to avoid cloning the descriptor - // if this type is already only an enclosing type. - isDirty = true - newDescriptor = maybeClone(descriptor, b.options) - sourcePathsRemap.markNoComment(sourcePath) - sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) - sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) - sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) - sourcePathsRemap.markDeleted(append(sourcePath, messageReservedRangesTag)) - sourcePathsRemap.markDeleted(append(sourcePath, messageReservedNamesTag)) - newDescriptor.Field = nil - newDescriptor.OneofDecl = nil - newDescriptor.ExtensionRange = nil - newDescriptor.ReservedRange = nil - newDescriptor.ReservedName = nil + if len(descriptor.Field) > 0 || + len(descriptor.OneofDecl) > 0 || + len(descriptor.ExtensionRange) > 0 || + len(descriptor.ReservedRange) > 0 || + len(descriptor.ReservedName) > 0 { + // Clear unnecessary fields. + isDirty = true + newDescriptor = maybeClone(descriptor, b.options) + sourcePathsRemap.markNoComment(sourcePath) + sourcePathsRemap.markDeleted(append(sourcePath, messageFieldsTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageOneofsTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageExtensionRangesTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageReservedRangesTag)) + sourcePathsRemap.markDeleted(append(sourcePath, messageReservedNamesTag)) + newDescriptor.Field = nil + newDescriptor.OneofDecl = nil + newDescriptor.ExtensionRange = nil + newDescriptor.ReservedRange = nil + newDescriptor.ReservedName = nil + } } else { newFields, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageFieldsTag), descriptor.GetField(), b.remapField, b.options) if err != nil { From 5035d617f89e8d872ffbd7e16a2c4280a8077763 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Wed, 19 Mar 2025 22:04:42 +0100 Subject: [PATCH 62/80] Simplify default check --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index e2cd938cda..9725857d51 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -238,9 +238,7 @@ func FilterImage(image bufimage.Image, options ...ImageFilterOption) (bufimage.I option(filterOptions) } // Check for defaults that would result in no filtering. - if filterOptions.includeCustomOptions && - filterOptions.includeKnownExtensions && - len(filterOptions.excludeTypes) == 0 && + if len(filterOptions.excludeTypes) == 0 && len(filterOptions.includeTypes) == 0 { return image, nil } From 12ce214b7f4c703fbbfb25055ebf4b723f8fe752 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 14:39:11 +0100 Subject: [PATCH 63/80] Fix include method but exclude input or output --- .../bufimage/bufimageutil/bufimageutil.go | 31 ++- .../bufimageutil/bufimageutil_test.go | 14 +- .../pkg.FooService.exclude-method-types.txtar | 191 ++++++++++++++++++ 3 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude-method-types.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 9725857d51..7ed8f464c9 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -478,6 +478,22 @@ func (t *transitiveClosure) addElement( case *descriptorpb.ServiceDescriptorProto: for _, method := range typedDescriptor.GetMethod() { + inputName := protoreflect.FullName(strings.TrimPrefix(method.GetInputType(), ".")) + inputInfo, ok := imageIndex.ByName[inputName] + if !ok { + return fmt.Errorf("missing %q", inputName) + } + outputName := protoreflect.FullName(strings.TrimPrefix(method.GetOutputType(), ".")) + outputInfo, ok := imageIndex.ByName[outputName] + if !ok { + return fmt.Errorf("missing %q", outputName) + } + inputMode, outputMode := t.elements[inputInfo.element], t.elements[outputInfo.element] + if inputMode == inclusionModeExcluded || outputMode == inclusionModeExcluded { + // The input or ouptut is excluded, so this method is also excluded. + t.elements[inputInfo.element] = inclusionModeExcluded + continue + } if err := t.addElement(method, "", false, imageIndex, opts); err != nil { return err } @@ -489,15 +505,22 @@ func (t *transitiveClosure) addElement( if !ok { return fmt.Errorf("missing %q", inputName) } - if err := t.addElement(inputInfo.element, descriptorInfo.file.Path(), false, imageIndex, opts); err != nil { - return err - } - outputName := protoreflect.FullName(strings.TrimPrefix(typedDescriptor.GetOutputType(), ".")) outputInfo, ok := imageIndex.ByName[outputName] if !ok { return fmt.Errorf("missing %q", outputName) } + if inputMode := t.elements[inputInfo.element]; inputMode == inclusionModeExcluded { + // The input is excluded, it's an error to include the method. + return fmt.Errorf("cannot include method %q as the input type %q is excluded", descriptorInfo.fullName, inputInfo.fullName) + } + if outputMode := t.elements[outputInfo.element]; outputMode == inclusionModeExcluded { + // The output is excluded, it's an error to include the method. + return fmt.Errorf("cannot include method %q as the output type %q is excluded", descriptorInfo.fullName, outputInfo.fullName) + } + if err := t.addElement(inputInfo.element, descriptorInfo.file.Path(), false, imageIndex, opts); err != nil { + return err + } if err := t.addElement(outputInfo.element, descriptorInfo.file.Path(), false, imageIndex, opts); err != nil { return err } diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 3d1488ed45..3337a41360 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -109,6 +109,18 @@ func TestTypes(t *testing.T) { t.Parallel() runDiffTest(t, "testdata/options", "pkg.FooService.mixed.txtar", WithIncludeTypes("pkg.FooService"), WithExcludeTypes("pkg.FooService.Do")) }) + t.Run("include-service-exclude-method-types", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/options", "pkg.FooService.exclude-method-types.txtar", WithIncludeTypes("pkg.FooService"), WithExcludeTypes("pkg.Empty")) + }) + t.Run("include-method-exclude-method-types", func(t *testing.T) { + t.Parallel() + _, image, err := getImage(context.Background(), slogtestext.NewLogger(t), "testdata/options", bufimage.WithExcludeSourceCodeInfo()) + require.NoError(t, err) + _, err = FilterImage(image, WithIncludeTypes("pkg.FooService", "pkg.FooService.Do"), WithExcludeTypes("pkg.Empty")) + require.Error(t, err) + assert.ErrorContains(t, err, "cannot include method \"pkg.FooService.Do\"") + }) } func TestNesting(t *testing.T) { @@ -461,7 +473,7 @@ func runDiffTest(t *testing.T, testdataDir string, expectedFile string, opts ... func runFilterImage(t *testing.T, image bufimage.Image, opts ...ImageFilterOption) []byte { filteredImage, err := FilterImage(image, opts...) require.NoError(t, err) - assert.NotNil(t, image) + assert.NotNil(t, filteredImage) assert.True(t, imageIsDependencyOrdered(filteredImage), "image files not in dependency order") // Convert the filtered image back to a proto image and then back to an image to ensure that the diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude-method-types.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude-method-types.txtar new file mode 100644 index 0000000000..d217c0ee80 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/options/pkg.FooService.exclude-method-types.txtar @@ -0,0 +1,191 @@ +-- a.proto -- +syntax = "proto2"; +package pkg; +import "options.proto"; +option (UsedOption.file_baz) = "str"; +option (UsedOption.file_foo) = { foo: "str" }; +service FooService { + option (service_baz) = "str"; + option (service_foo) = { foo: "str" }; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional bool java_multiple_files = 10 [default = false]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated = true]; + optional bool deprecated = 23 [default = false]; + optional bool java_string_check_utf8 = 27 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + optional FeatureSet features = 50; + repeated UninterpretedOption uninterpreted_option = 999; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + extensions 1000 to max; + reserved 38, 42; + reserved "php_generic_services"; +} +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + optional FeatureSet features = 34; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} +-- options.proto -- +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; +message UsedOption { + string foo = 1; + extend google.protobuf.FileOptions { + UsedOption file_foo = 50000; + string file_baz = 50002; + } +} +extend google.protobuf.ServiceOptions { + UsedOption service_foo = 50000; + string service_baz = 50002; +} From f5cfcd523e284d1ca615469c3e33ffce0a5254ae Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 15:51:08 +0100 Subject: [PATCH 64/80] Fix empty oneofs --- .../bufimage/bufimageutil/bufimageutil.go | 12 +- .../bufimageutil/bufimageutil_test.go | 12 ++ .../bufimage/bufimageutil/image_filter.go | 4 + .../bufimageutil/testdata/oneofs/a.proto | 31 ++++ .../testdata/oneofs/options.proto | 0 .../testdata/oneofs/pkg.Foo.exclude-bar.txtar | 148 +++++++++++++++++ .../oneofs/pkg.Foo.exclude-partial.txtar | 156 ++++++++++++++++++ 7 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-bar.txtar create mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-partial.txtar diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 7ed8f464c9..859a0a7ed3 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -443,6 +443,7 @@ func (t *transitiveClosure) addElement( } case *descriptorpb.DescriptorProto: + oneofFieldCounts := make([]int, len(typedDescriptor.GetOneofDecl())) // Options and types for all fields for _, field := range typedDescriptor.GetField() { isIncluded, err := t.addFieldType(field, descriptorInfo.file.Path(), imageIndex, opts) @@ -452,12 +453,21 @@ func (t *transitiveClosure) addElement( if !isIncluded { continue } + if index := field.OneofIndex; index != nil { + oneofFieldCounts[*index]++ + } if err := t.exploreCustomOptions(field, referrerFile, imageIndex, opts); err != nil { return err } } // Options for all oneofs in this message - for _, oneOfDescriptor := range typedDescriptor.GetOneofDecl() { + for index, oneOfDescriptor := range typedDescriptor.GetOneofDecl() { + if oneofFieldCounts[index] == 0 { + // An empty oneof is not a valid protobuf construct, so we can + // safely exclude it. + t.elements[oneOfDescriptor] = inclusionModeExcluded + continue + } if err := t.exploreCustomOptions(oneOfDescriptor, descriptorInfo.file.Path(), imageIndex, opts); err != nil { return err } diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 3337a41360..f2a19f5137 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -176,6 +176,18 @@ func TestNesting(t *testing.T) { }) } +func TestOneof(t *testing.T) { + t.Parallel() + t.Run("exclude-partial", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/oneofs", "pkg.Foo.exclude-partial.txtar", WithIncludeTypes("pkg.Foo"), WithExcludeTypes("pkg.FooEnum", "pkg.Bar.BarNested")) + }) + t.Run("exclude-bar", func(t *testing.T) { + t.Parallel() + runDiffTest(t, "testdata/oneofs", "pkg.Foo.exclude-bar.txtar", WithIncludeTypes("pkg.Foo"), WithExcludeTypes("pkg.FooEnum", "pkg.Bar")) + }) +} + func TestOptions(t *testing.T) { t.Parallel() t.Run("include_option", func(t *testing.T) { diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 59fa1df4fa..05853f6d19 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -436,6 +436,10 @@ func (b *sourcePathsBuilder) remapOneof( sourcePath protoreflect.SourcePath, oneof *descriptorpb.OneofDescriptorProto, ) (*descriptorpb.OneofDescriptorProto, bool, error) { + if mode, ok := b.closure.elements[oneof]; ok && mode == inclusionModeExcluded { + // Oneofs can be explicitly excluded when all fields for it are excluded. + return nil, true, nil + } options, changed, err := remapMessage(nil, append(sourcePath, oneofOptionsTag), oneof.GetOptions(), b.remapOptions) if err != nil { return nil, false, err diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto new file mode 100644 index 0000000000..ef76aa03d2 --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; +package pkg; +import "google/protobuf/descriptor.proto"; + +message Foo { + oneof oneofFoo { + option (oneof_foo) = "str"; + string foo = 1; + FooEnum foo_enum = 2; + } + oneof oneofBar { + option (oneof_bar) = "str"; + Bar bar = 3; + Bar.BarNested bar_nested = 4; + } +} + +enum FooEnum { + FOO_ENUM_X = 0; + FOO_ENUM_Y = 1; +} +message Bar { + string bar = 1; + message BarNested { + string bar = 1; + } +} +extend google.protobuf.OneofOptions { + optional string oneof_foo = 50000; + optional string oneof_bar = 50001; +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto new file mode 100644 index 0000000000..e69de29bb2 diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-bar.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-bar.txtar new file mode 100644 index 0000000000..acaf7c9a2a --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-bar.txtar @@ -0,0 +1,148 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "google/protobuf/descriptor.proto"; +message Foo { + oneof oneofFoo { + option (oneof_foo) = "str"; + string foo = 1; + } +} +extend google.protobuf.OneofOptions { + string oneof_foo = 50000; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-partial.txtar b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-partial.txtar new file mode 100644 index 0000000000..e10eaf1a5c --- /dev/null +++ b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/pkg.Foo.exclude-partial.txtar @@ -0,0 +1,156 @@ +-- a.proto -- +syntax = "proto3"; +package pkg; +import "google/protobuf/descriptor.proto"; +message Bar { + string bar = 1; +} +message Foo { + oneof oneofFoo { + option (oneof_foo) = "str"; + string foo = 1; + } + oneof oneofBar { + option (oneof_bar) = "str"; + Bar bar = 3; + } +} +extend google.protobuf.OneofOptions { + string oneof_foo = 50000; + string oneof_bar = 50001; +} +-- google/protobuf/descriptor.proto -- +syntax = "proto2"; +package google.protobuf; +option cc_enable_arenas = true; +option csharp_namespace = "Google.Protobuf.Reflection"; +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_outer_classname = "DescriptorProtos"; +option java_package = "com.google.protobuf"; +option objc_class_prefix = "GPB"; +option optimize_for = SPEED; +message FeatureSet { + optional FieldPresence field_presence = 1 [ + edition_defaults = { value: "EXPLICIT", edition: EDITION_LEGACY }, + edition_defaults = { value: "IMPLICIT", edition: EDITION_PROTO3 }, + edition_defaults = { value: "EXPLICIT", edition: EDITION_2023 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional EnumType enum_type = 2 [ + edition_defaults = { value: "CLOSED", edition: EDITION_LEGACY }, + edition_defaults = { value: "OPEN", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + edition_defaults = { value: "EXPANDED", edition: EDITION_LEGACY }, + edition_defaults = { value: "PACKED", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional Utf8Validation utf8_validation = 4 [ + edition_defaults = { value: "NONE", edition: EDITION_LEGACY }, + edition_defaults = { value: "VERIFY", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional MessageEncoding message_encoding = 5 [ + edition_defaults = { value: "LENGTH_PREFIXED", edition: EDITION_LEGACY }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE + ]; + optional JsonFormat json_format = 6 [ + edition_defaults = { value: "LEGACY_BEST_EFFORT", edition: EDITION_LEGACY }, + edition_defaults = { value: "ALLOW", edition: EDITION_PROTO3 }, + feature_support = { edition_introduced: EDITION_2023 }, + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE + ]; + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { + number: 1002, + full_name: ".pb.go", + type: ".pb.GoFeatures" + }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + extensions 9995 to 9999, 10000; + reserved 999; +} +message OneofOptions { + optional FeatureSet features = 1; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} +message UninterpretedOption { + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } +} From 01f7b0956d454b795393098b8104fcb7ddd7e379 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 15:55:55 +0100 Subject: [PATCH 65/80] Fix spacing --- .../bufimageutil/testdata/oneofs/a.proto | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto index ef76aa03d2..c3bc638414 100644 --- a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto +++ b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/a.proto @@ -3,16 +3,16 @@ package pkg; import "google/protobuf/descriptor.proto"; message Foo { - oneof oneofFoo { - option (oneof_foo) = "str"; + oneof oneofFoo { + option (oneof_foo) = "str"; string foo = 1; FooEnum foo_enum = 2; - } - oneof oneofBar { - option (oneof_bar) = "str"; + } + oneof oneofBar { + option (oneof_bar) = "str"; Bar bar = 3; Bar.BarNested bar_nested = 4; - } + } } enum FooEnum { @@ -26,6 +26,6 @@ message Bar { } } extend google.protobuf.OneofOptions { - optional string oneof_foo = 50000; - optional string oneof_bar = 50001; + optional string oneof_foo = 50000; + optional string oneof_bar = 50001; } From f5398e560c95aa839dfd60e6bd9f01838901f65b Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 15:57:01 +0100 Subject: [PATCH 66/80] Cleanup --- .../bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto diff --git a/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto b/private/bufpkg/bufimage/bufimageutil/testdata/oneofs/options.proto deleted file mode 100644 index e69de29bb2..0000000000 From 62b44021f210dce970f53ca17ccd78037fb742a1 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 17:41:54 +0100 Subject: [PATCH 67/80] Check extendee valid for extension inclusions --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 11 +++++++++++ .../bufpkg/bufimage/bufimageutil/bufimageutil_test.go | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 859a0a7ed3..0f6a8cc38d 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -358,6 +358,17 @@ func (t *transitiveClosure) includeType( if mode := t.elements[descriptorInfo.element]; mode == inclusionModeExcluded { return fmt.Errorf("inclusion of excluded type %q", typeName) } + // If an extension field, check if the extendee is excluded. + if field, ok := descriptorInfo.element.(*descriptorpb.FieldDescriptorProto); ok && field.Extendee != nil { + extendeeName := protoreflect.FullName(strings.TrimPrefix(field.GetExtendee(), ".")) + extendeeInfo, ok := imageIndex.ByName[extendeeName] + if !ok { + return fmt.Errorf("missing %q", extendeeName) + } + if mode := t.elements[extendeeInfo.element]; mode == inclusionModeExcluded { + return fmt.Errorf("cannot include extension field %q as the extendee type %q is excluded", typeName, extendeeName) + } + } if err := t.addElement(descriptorInfo.element, "", false, imageIndex, options); err != nil { return fmt.Errorf("inclusion of type %q: %w", typeName, err) } diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index f2a19f5137..2483809674 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -121,6 +121,15 @@ func TestTypes(t *testing.T) { require.Error(t, err) assert.ErrorContains(t, err, "cannot include method \"pkg.FooService.Do\"") }) + + t.Run("include-extension-exclude-extendee", func(t *testing.T) { + t.Parallel() + _, image, err := getImage(context.Background(), slogtestext.NewLogger(t), "testdata/options", bufimage.WithExcludeSourceCodeInfo()) + require.NoError(t, err) + _, err = FilterImage(image, WithIncludeTypes("pkg.extension"), WithExcludeTypes("pkg.Foo")) + require.Error(t, err) + assert.ErrorContains(t, err, "cannot include extension field \"pkg.extension\" as the extendee type \"pkg.Foo\" is excluded") + }) } func TestNesting(t *testing.T) { From 1696ccaaaee4f11406233a7a8244f911299775b7 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 19:59:17 +0100 Subject: [PATCH 68/80] Use IncludeTypes naming --- private/buf/bufctl/controller.go | 2 +- private/buf/bufctl/option.go | 6 +++--- private/buf/cmd/buf/command/build/build.go | 2 +- private/buf/cmd/buf/command/generate/generate.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 9669d25330..b829f66302 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -1337,7 +1337,7 @@ func filterImage( if functionOptions.imageExcludeImports { newImage = bufimage.ImageWithoutImports(newImage) } - includeTypes := functionOptions.imageTypes + includeTypes := functionOptions.imageIncludeTypes excludeTypes := functionOptions.imageExcludeTypes if len(includeTypes) > 0 || len(excludeTypes) > 0 { newImage, err = bufimageutil.FilterImage( diff --git a/private/buf/bufctl/option.go b/private/buf/bufctl/option.go index 5a0d2d419b..8adae0c288 100644 --- a/private/buf/bufctl/option.go +++ b/private/buf/bufctl/option.go @@ -66,9 +66,9 @@ func WithImageExcludeImports(imageExcludeImports bool) FunctionOption { } } -func WithImageTypes(imageTypes []string) FunctionOption { +func WithImageIncludeTypes(imageTypes []string) FunctionOption { return func(functionOptions *functionOptions) { - functionOptions.imageTypes = imageTypes + functionOptions.imageIncludeTypes = imageTypes } } @@ -139,7 +139,7 @@ type functionOptions struct { targetExcludePaths []string imageExcludeSourceInfo bool imageExcludeImports bool - imageTypes []string + imageIncludeTypes []string imageExcludeTypes []string imageAsFileDescriptorSet bool configOverride string diff --git a/private/buf/cmd/buf/command/build/build.go b/private/buf/cmd/buf/command/build/build.go index c3e4187270..2e70e7e9dd 100644 --- a/private/buf/cmd/buf/command/build/build.go +++ b/private/buf/cmd/buf/command/build/build.go @@ -158,7 +158,7 @@ func run( bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), bufctl.WithImageExcludeSourceInfo(flags.ExcludeSourceInfo), bufctl.WithImageExcludeImports(flags.ExcludeImports), - bufctl.WithImageTypes(flags.Types), + bufctl.WithImageIncludeTypes(flags.Types), bufctl.WithConfigOverride(flags.Config), ) if err != nil { diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 8c9163e74c..3beaf10098 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -635,7 +635,7 @@ func getInputImages( input, bufctl.WithConfigOverride(moduleConfigOverride), bufctl.WithTargetPaths(targetPathsOverride, excludePathsOverride), - bufctl.WithImageTypes(includeTypes), + bufctl.WithImageIncludeTypes(includeTypes), ) if err != nil { return nil, err @@ -665,7 +665,7 @@ func getInputImages( inputConfig, bufctl.WithConfigOverride(moduleConfigOverride), bufctl.WithTargetPaths(targetPaths, excludePaths), - bufctl.WithImageTypes(includeTypes), + bufctl.WithImageIncludeTypes(includeTypes), bufctl.WithImageExcludeTypes(excludeTypes), ) if err != nil { From 584bde7d4c73ce6a6881e71ab4c59cf6f5b7787a Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 20 Mar 2025 20:01:29 +0100 Subject: [PATCH 69/80] Use includeTypes names --- .../bufconfig/generate_plugin_config.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 13cd92327b..97c9e77d97 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -127,7 +127,7 @@ func NewRemoteGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, - types []string, + includeTypes []string, excludeTypes []string, revision int, ) (GeneratePluginConfig, error) { @@ -137,7 +137,7 @@ func NewRemoteGeneratePluginConfig( opt, includeImports, includeWKT, - types, + includeTypes, excludeTypes, revision, ) @@ -509,7 +509,7 @@ func newRemoteGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, - types []string, + includeTypes []string, excludeTypes []string, revision int, ) (*generatePluginConfig, error) { @@ -532,7 +532,7 @@ func newRemoteGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, - includeTypes: types, + includeTypes: includeTypes, excludeTypes: excludeTypes, }, nil } @@ -543,7 +543,7 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, - types []string, + includeTypes []string, excludeTypes []string, strategy *GenerateStrategy, ) (*generatePluginConfig, error) { @@ -558,7 +558,7 @@ func newLocalOrProtocBuiltinGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, - includeTypes: types, + includeTypes: includeTypes, excludeTypes: excludeTypes, }, nil } @@ -569,7 +569,7 @@ func newLocalGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, - types []string, + includeTypes []string, excludeTypes []string, strategy *GenerateStrategy, path []string, @@ -589,7 +589,7 @@ func newLocalGeneratePluginConfig( opts: opt, includeImports: includeImports, includeWKT: includeWKT, - includeTypes: types, + includeTypes: includeTypes, excludeTypes: excludeTypes, }, nil } @@ -600,7 +600,7 @@ func newProtocBuiltinGeneratePluginConfig( opt []string, includeImports bool, includeWKT bool, - types []string, + includeTypes []string, excludeTypes []string, strategy *GenerateStrategy, protocPath []string, @@ -617,7 +617,7 @@ func newProtocBuiltinGeneratePluginConfig( strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, - includeTypes: types, + includeTypes: includeTypes, excludeTypes: excludeTypes, }, nil } From 304f0afe6e6d554354ef28eea37d2579bd22ae84 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 25 Mar 2025 16:23:12 +0100 Subject: [PATCH 70/80] Set options as unrecognized bytes --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 05853f6d19..addc5c7180 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -564,13 +564,8 @@ func (b *sourcePathsBuilder) remapOptions( numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { if !b.closure.hasOption(fd, b.imageIndex, b.options) { - // Remove this option. - optionPath := append(optionsPath, int32(fd.Number())) - sourcePathsRemap.markDeleted(optionPath) - if newOptions == nil { - newOptions = maybeClone(optionsMessage, b.options).ProtoReflect() - } - newOptions.Clear(fd) + // Options don't need to be removed and can just be left + // as unrecognized bytes. return true } numFieldsToKeep++ From 698e350d478e9cb2d937380a2921b31f671055e1 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 25 Mar 2025 19:13:23 +0100 Subject: [PATCH 71/80] Revert "Set options as unrecognized bytes" This reverts commit 304f0afe6e6d554354ef28eea37d2579bd22ae84. --- private/bufpkg/bufimage/bufimageutil/image_filter.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index addc5c7180..05853f6d19 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -564,8 +564,13 @@ func (b *sourcePathsBuilder) remapOptions( numFieldsToKeep := 0 options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { if !b.closure.hasOption(fd, b.imageIndex, b.options) { - // Options don't need to be removed and can just be left - // as unrecognized bytes. + // Remove this option. + optionPath := append(optionsPath, int32(fd.Number())) + sourcePathsRemap.markDeleted(optionPath) + if newOptions == nil { + newOptions = maybeClone(optionsMessage, b.options).ProtoReflect() + } + newOptions.Clear(fd) return true } numFieldsToKeep++ From d57a7d9c35c3903d6ed30f3b56bb8b9f040eafd0 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Mar 2025 17:43:23 +0100 Subject: [PATCH 72/80] Move import check to closure --- .../bufimage/bufimageutil/bufimageutil.go | 10 +++++++--- .../bufimage/bufimageutil/image_filter.go | 20 +++++-------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 0f6a8cc38d..cd51420bb4 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -403,6 +403,12 @@ func (t *transitiveClosure) includeType( } func (t *transitiveClosure) addImport(fromPath, toPath string) { + if _, ok := t.imports[toPath]; !ok { + t.imports[toPath] = nil // mark as seen + } + if fromPath == "" { + return // the base included type, not imported + } if fromPath == toPath { return // no need for a file to import itself } @@ -422,9 +428,7 @@ func (t *transitiveClosure) addElement( opts *imageFilterOptions, ) error { descriptorInfo := imageIndex.ByDescriptor[descriptor] - if referrerFile != "" { - t.addImport(referrerFile, descriptorInfo.file.Path()) - } + t.addImport(referrerFile, descriptorInfo.file.Path()) if existingMode, ok := t.elements[descriptor]; ok && existingMode != inclusionModeEnclosing { if existingMode == inclusionModeImplicit && !impliedByCustomOption { // upgrade from implied to explicitly part of closure diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 05853f6d19..9030144a92 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -67,17 +67,13 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im // Loop over image files in revserse DAG order. Imports that are no longer // imported by a previous file are dropped from the image. - imageFiles := image.Files() dirty := false newImageFiles := make([]bufimage.ImageFile, 0, len(image.Files())) - importsByFilePath := make(map[string]struct{}) - for _, imageFile := range slices.Backward(imageFiles) { + for _, imageFile := range slices.Backward(image.Files()) { imageFilePath := imageFile.Path() - _, isFileImported := importsByFilePath[imageFilePath] - // Check if this import is still used. If allowImportedTypes is true, we - // must check the imported file to see if it is used. - if imageFile.IsImport() && !options.allowImportedTypes && !isFileImported { - continue + // Check if the file is used. + if _, ok := closure.imports[imageFilePath]; !ok { + continue // Filtered out. } newImageFile, err := filterImageFile( imageFile, @@ -90,13 +86,7 @@ func filterImage(image bufimage.Image, options *imageFilterOptions) (bufimage.Im } dirty = dirty || newImageFile != imageFile if newImageFile == nil { - if isFileImported { - return nil, fmt.Errorf("imported file %q was filtered out", imageFilePath) - } - continue // Filtered out. - } - for _, filePath := range newImageFile.FileDescriptorProto().Dependency { - importsByFilePath[filePath] = struct{}{} + return nil, fmt.Errorf("imported file %q was filtered out", imageFilePath) } newImageFiles = append(newImageFiles, newImageFile) } From 7b1f6bfd66d69fab68d36ebea1f1908eeb80e8b9 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Mar 2025 17:47:49 +0100 Subject: [PATCH 73/80] Fix docs --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 2 +- private/bufpkg/bufimage/bufimageutil/image_filter.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index cd51420bb4..1d53c9ab0b 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -336,7 +336,7 @@ func (t *transitiveClosure) hasOption( return false case inclusionModeUnknown: // True as option type is not explicitly excluded. - // Occurs on first transversal when adding included types. + // Occurs on first traversal when adding included types. return true default: return false diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 9030144a92..27a6087eab 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -427,7 +427,7 @@ func (b *sourcePathsBuilder) remapOneof( oneof *descriptorpb.OneofDescriptorProto, ) (*descriptorpb.OneofDescriptorProto, bool, error) { if mode, ok := b.closure.elements[oneof]; ok && mode == inclusionModeExcluded { - // Oneofs can be explicitly excluded when all fields for it are excluded. + // Oneofs are implicitly excluded when all of its fields types are excluded. return nil, true, nil } options, changed, err := remapMessage(nil, append(sourcePath, oneofOptionsTag), oneof.GetOptions(), b.remapOptions) From d327c23f59e2ebc2e6ee9c58510ab629d190cbef Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Mar 2025 17:49:27 +0100 Subject: [PATCH 74/80] Add range check --- private/bufpkg/bufimage/bufimageutil/bufimageutil.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 1d53c9ab0b..f773348132 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -469,7 +469,11 @@ func (t *transitiveClosure) addElement( continue } if index := field.OneofIndex; index != nil { - oneofFieldCounts[*index]++ + index := *index + if index < 0 || int(index) >= len(oneofFieldCounts) { + return fmt.Errorf("invalid oneof index %d for field %q", index, field.GetName()) + } + oneofFieldCounts[index]++ } if err := t.exploreCustomOptions(field, referrerFile, imageIndex, opts); err != nil { return err From b4c1f56931d6cae7d0661fc6d3e53afbf8fb5e28 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 27 Mar 2025 18:29:52 +0100 Subject: [PATCH 75/80] Remove options remap --- .../bufimage/bufimageutil/image_filter.go | 148 +----------------- 1 file changed, 4 insertions(+), 144 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 27a6087eab..164bbe7e69 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -199,11 +199,6 @@ func (b *sourcePathsBuilder) remapFileDescriptor( return nil, false, err } isDirty = isDirty || changed - newOptions, changed, err := remapMessage(nil, append(sourcePath, fileOptionsTag), fileDescriptor.Options, b.remapOptions) - if err != nil { - return nil, false, err - } - isDirty = isDirty || changed if !isDirty { return fileDescriptor, false, nil @@ -214,7 +209,6 @@ func (b *sourcePathsBuilder) remapFileDescriptor( newFileDescriptor.EnumType = newEnums newFileDescriptor.Service = newServices newFileDescriptor.Extension = newExtensions - newFileDescriptor.Options = newOptions // Fix the imports to remove any that are no longer used. importsRequired := b.closure.imports[fileDescriptor.GetName()] @@ -310,16 +304,10 @@ func (b *sourcePathsBuilder) remapDescriptor( return nil, false, err } isDirty = isDirty || changed - newExtensionRange, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionRangesTag), descriptor.ExtensionRange, b.remapExtensionRange, b.options) - if err != nil { - return nil, false, err - } - isDirty = isDirty || changed if isDirty { newDescriptor = maybeClone(descriptor, b.options) newDescriptor.Field = newFields newDescriptor.OneofDecl = newOneofs - newDescriptor.ExtensionRange = newExtensionRange } } newExtensions, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, messageExtensionsTag), descriptor.GetExtension(), b.remapField, b.options) @@ -337,11 +325,6 @@ func (b *sourcePathsBuilder) remapDescriptor( return nil, false, err } isDirty = isDirty || changed - newOptions, changed, err := remapMessage(nil, append(sourcePath, messageOptionsTag), descriptor.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - isDirty = isDirty || changed if !isDirty { return descriptor, false, nil @@ -352,27 +335,9 @@ func (b *sourcePathsBuilder) remapDescriptor( newDescriptor.Extension = newExtensions newDescriptor.NestedType = newDescriptors newDescriptor.EnumType = newEnums - newDescriptor.Options = newOptions return newDescriptor, true, nil } -func (b *sourcePathsBuilder) remapExtensionRange( - sourcePathsRemap *sourcePathsRemapTrie, - sourcePath protoreflect.SourcePath, - extensionRange *descriptorpb.DescriptorProto_ExtensionRange, -) (*descriptorpb.DescriptorProto_ExtensionRange, bool, error) { - newOptions, changed, err := remapMessage(nil, append(sourcePath, extensionRangeOptionsTag), extensionRange.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - if !changed { - return extensionRange, false, nil - } - newExtensionRange := maybeClone(extensionRange, b.options) - newExtensionRange.Options = newOptions - return newExtensionRange, true, nil -} - func (b *sourcePathsBuilder) remapEnum( sourcePathsRemap *sourcePathsRemapTrie, sourcePath protoreflect.SourcePath, @@ -382,43 +347,7 @@ func (b *sourcePathsBuilder) remapEnum( // The type is excluded, enum values cannot be excluded individually. return nil, true, nil } - var isDirty bool - newOptions, changed, err := remapMessage(nil, append(sourcePath, enumOptionsTag), enum.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - isDirty = changed - - // Walk the enum values. - newEnumValues, changed, err := remapSlice(sourcePathsRemap, append(sourcePath, enumValuesTag), enum.Value, b.remapEnumValue, b.options) - if err != nil { - return nil, false, err - } - isDirty = isDirty || changed - if !isDirty { - return enum, true, nil - } - newEnum := maybeClone(enum, b.options) - newEnum.Options = newOptions - newEnum.Value = newEnumValues - return newEnum, true, nil -} - -func (b *sourcePathsBuilder) remapEnumValue( - sourcePathsRemap *sourcePathsRemapTrie, - sourcePath protoreflect.SourcePath, - enumValue *descriptorpb.EnumValueDescriptorProto, -) (*descriptorpb.EnumValueDescriptorProto, bool, error) { - newOptions, changed, err := remapMessage(nil, append(sourcePath, enumValueOptionsTag), enumValue.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - if !changed { - return enumValue, false, nil - } - newEnumValue := maybeClone(enumValue, b.options) - newEnumValue.Options = newOptions - return newEnumValue, true, nil + return enum, false, nil } func (b *sourcePathsBuilder) remapOneof( @@ -430,16 +359,7 @@ func (b *sourcePathsBuilder) remapOneof( // Oneofs are implicitly excluded when all of its fields types are excluded. return nil, true, nil } - options, changed, err := remapMessage(nil, append(sourcePath, oneofOptionsTag), oneof.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - if !changed { - return oneof, false, nil - } - newOneof := maybeClone(oneof, b.options) - newOneof.Options = options - return newOneof, true, nil + return oneof, false, nil } func (b *sourcePathsBuilder) remapService( @@ -457,17 +377,11 @@ func (b *sourcePathsBuilder) remapService( return nil, false, err } isDirty = isDirty || changed - newOptions, changed, err := remapMessage(nil, append(sourcePath, serviceOptionsTag), service.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - isDirty = isDirty || changed if !isDirty { return service, false, nil } newService := maybeClone(service, b.options) newService.Method = newMethods - newService.Options = newOptions return newService, true, nil } @@ -479,16 +393,7 @@ func (b *sourcePathsBuilder) remapMethod( if !b.closure.hasType(method, b.options) { return nil, true, nil } - newOptions, changed, err := remapMessage(nil, append(sourcePath, methodOptionsTag), method.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - if !changed { - return method, false, nil - } - newMethod := maybeClone(method, b.options) - newMethod.Options = newOptions - return newMethod, true, nil + return method, false, nil } func (b *sourcePathsBuilder) remapField( @@ -529,52 +434,7 @@ func (b *sourcePathsBuilder) remapField( default: return nil, false, fmt.Errorf("unknown field type %d", field.GetType()) } - newOption, changed, err := remapMessage(nil, append(sourcePath, fieldOptionsTag), field.GetOptions(), b.remapOptions) - if err != nil { - return nil, false, err - } - if !changed { - return field, false, nil - } - newField := maybeClone(field, b.options) - newField.Options = newOption - return newField, true, nil -} - -func (b *sourcePathsBuilder) remapOptions( - sourcePathsRemap *sourcePathsRemapTrie, - optionsPath protoreflect.SourcePath, - optionsMessage proto.Message, -) (proto.Message, bool, error) { - if optionsMessage == nil { - return nil, false, nil - } - var newOptions protoreflect.Message - options := optionsMessage.ProtoReflect() - numFieldsToKeep := 0 - options.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { - if !b.closure.hasOption(fd, b.imageIndex, b.options) { - // Remove this option. - optionPath := append(optionsPath, int32(fd.Number())) - sourcePathsRemap.markDeleted(optionPath) - if newOptions == nil { - newOptions = maybeClone(optionsMessage, b.options).ProtoReflect() - } - newOptions.Clear(fd) - return true - } - numFieldsToKeep++ - return true - }) - if numFieldsToKeep == 0 { - // No options to keep. - sourcePathsRemap.markDeleted(optionsPath) - return nil, true, nil - } - if newOptions == nil { - return optionsMessage, false, nil - } - return newOptions.Interface(), true, nil + return field, false, nil } func remapMessage[T proto.Message]( From b6a92ee8b9d00f6ac1e3a543624e0a614e767bba Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 28 Mar 2025 01:08:22 +0100 Subject: [PATCH 76/80] Remap depencies to catch changes --- .../bufimage/bufimageutil/bufimageutil.go | 5 ++ .../bufimage/bufimageutil/image_filter.go | 66 +++++++++++++++---- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index f773348132..2e9cb9f1d6 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -1118,3 +1118,8 @@ func (o *orderedImports) keys() []string { } return keys } + +// len returns the number of paths in the ordered imports. +func (o *orderedImports) len() int { + return len(o.pathToIndex) +} diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index 164bbe7e69..dbcb7dabd7 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -199,7 +199,11 @@ func (b *sourcePathsBuilder) remapFileDescriptor( return nil, false, err } isDirty = isDirty || changed - + newDependencies, newPublicDependencies, newWeakDependencies, changed, err := b.remapDependencies(sourcePathsRemap, append(sourcePath, fileDependencyTag), fileDescriptor) + if err != nil { + return nil, false, err + } + isDirty = isDirty || changed if !isDirty { return fileDescriptor, false, nil } @@ -209,22 +213,55 @@ func (b *sourcePathsBuilder) remapFileDescriptor( newFileDescriptor.EnumType = newEnums newFileDescriptor.Service = newServices newFileDescriptor.Extension = newExtensions + newFileDescriptor.Dependency = newDependencies + newFileDescriptor.PublicDependency = newPublicDependencies + newFileDescriptor.WeakDependency = newWeakDependencies + return newFileDescriptor, true, nil +} + +func (b *sourcePathsBuilder) remapDependencies( + sourcePathsRemap *sourcePathsRemapTrie, + sourcePath protoreflect.SourcePath, + fileDescriptor *descriptorpb.FileDescriptorProto, +) ([]string, []int32, []int32, bool, error) { + dependencies := fileDescriptor.GetDependency() + publicDependencies := fileDescriptor.GetPublicDependency() + weakDependencies := fileDescriptor.GetWeakDependency() - // Fix the imports to remove any that are no longer used. + // Check if the imports need to be remapped. importsRequired := b.closure.imports[fileDescriptor.GetName()] + importsCount := 0 + if importsRequired != nil { + importsCount = importsRequired.len() + } + for _, importPath := range dependencies { + if importsRequired != nil && importsRequired.index(importPath) != -1 { + importsCount-- + } else { + importsCount = -1 + break + } + } + if importsCount == 0 && len(publicDependencies) == 0 { + // Imports match and no public dependencies. + return dependencies, publicDependencies, weakDependencies, false, nil + } indexFrom, indexTo := int32(0), int32(0) - newFileDescriptor.Dependency = nil + var newDependencies []string + if b.options.mutateInPlace { + newDependencies = dependencies[:0] + } dependencyPath := append(sourcePath, fileDependencyTag) - dependencyChanges := make([]int32, len(fileDescriptor.Dependency)) - for _, importPath := range fileDescriptor.Dependency { + dependencyChanges := make([]int32, len(dependencies)) + for _, importPath := range dependencies { path := append(dependencyPath, indexFrom) if importsRequired != nil && importsRequired.index(importPath) != -1 { dependencyChanges[indexFrom] = indexTo if indexTo != indexFrom { sourcePathsRemap.markMoved(path, indexTo) } - newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, importPath) + newDependencies = append(newDependencies, importPath) indexTo++ // delete them as we go, so we know which ones weren't in the list importsRequired.delete(importPath) @@ -235,17 +272,20 @@ func (b *sourcePathsBuilder) remapFileDescriptor( indexFrom++ } if importsRequired != nil { - newFileDescriptor.Dependency = append(newFileDescriptor.Dependency, importsRequired.keys()...) + newDependencies = append(newDependencies, importsRequired.keys()...) } - newFileDescriptor.PublicDependency = nil + // Pulbic dependencies are always removed on remapping. publicDependencyPath := append(sourcePath, filePublicDependencyTag) sourcePathsRemap.markDeleted(publicDependencyPath) - newFileDescriptor.WeakDependency = nil - if len(fileDescriptor.WeakDependency) > 0 { + var newWeakDependencies []int32 + if len(weakDependencies) > 0 { + if b.options.mutateInPlace { + newWeakDependencies = weakDependencies[:0] + } weakDependencyPath := append(sourcePath, fileWeakDependencyTag) - for _, indexFrom := range fileDescriptor.WeakDependency { + for _, indexFrom := range weakDependencies { path := append(weakDependencyPath, indexFrom) indexTo := dependencyChanges[indexFrom] if indexTo == -1 { @@ -254,11 +294,11 @@ func (b *sourcePathsBuilder) remapFileDescriptor( if indexTo != indexFrom { sourcePathsRemap.markMoved(path, indexTo) } - newFileDescriptor.WeakDependency = append(newFileDescriptor.WeakDependency, indexTo) + newWeakDependencies = append(newWeakDependencies, indexTo) } } } - return newFileDescriptor, true, nil + return newDependencies, nil, newWeakDependencies, true, nil } func (b *sourcePathsBuilder) remapDescriptor( From 412b940abc9b2ecdf7e0cb070d0189a22124bbcb Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 28 Mar 2025 01:10:15 +0100 Subject: [PATCH 77/80] Fix race on bufimage.NewImageForProto --- private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 2483809674..6f8bbca33a 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -26,6 +26,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" + imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/protoencoding" "github.com/bufbuild/buf/private/pkg/slicesext" @@ -501,6 +502,8 @@ func runFilterImage(t *testing.T, image bufimage.Image, opts ...ImageFilterOptio // image is still valid after filtering. protoImage, err := bufimage.ImageToProtoImage(filteredImage) require.NoError(t, err) + // Clone here as `bufimage.NewImageForProto` mutates protoImage. + protoImage, _ = proto.Clone(protoImage).(*imagev1.Image) // Safe to assert. filteredImage, err = bufimage.NewImageForProto(protoImage) require.NoError(t, err) From c6b0e6abc90df125875ca9845f0c822072fbf583 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 28 Mar 2025 09:23:34 +0100 Subject: [PATCH 78/80] Fix source paths remaps for deps --- .../bufimage/bufimageutil/image_filter.go | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/image_filter.go b/private/bufpkg/bufimage/bufimageutil/image_filter.go index dbcb7dabd7..00335ea863 100644 --- a/private/bufpkg/bufimage/bufimageutil/image_filter.go +++ b/private/bufpkg/bufimage/bufimageutil/image_filter.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/syserror" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" @@ -119,6 +120,9 @@ func filterImageFile( return nil, nil // Filtered out. } if !changed { + if len(sourcePathsRemap) > 0 { + return nil, syserror.Newf("unexpected %d sourcePathsRemaps", len(sourcePathsRemap)) + } return imageFile, nil // No changes required. } @@ -199,7 +203,7 @@ func (b *sourcePathsBuilder) remapFileDescriptor( return nil, false, err } isDirty = isDirty || changed - newDependencies, newPublicDependencies, newWeakDependencies, changed, err := b.remapDependencies(sourcePathsRemap, append(sourcePath, fileDependencyTag), fileDescriptor) + newDependencies, newPublicDependencies, newWeakDependencies, changed, err := b.remapDependencies(sourcePathsRemap, sourcePath, fileDescriptor) if err != nil { return nil, false, err } @@ -477,27 +481,6 @@ func (b *sourcePathsBuilder) remapField( return field, false, nil } -func remapMessage[T proto.Message]( - sourcePathsRemap *sourcePathsRemapTrie, - sourcePath protoreflect.SourcePath, - message T, - remapMessage func(*sourcePathsRemapTrie, protoreflect.SourcePath, proto.Message) (proto.Message, bool, error), -) (T, bool, error) { - var zeroValue T - newMessageOpaque, changed, err := remapMessage(sourcePathsRemap, sourcePath, message) - if err != nil { - return zeroValue, false, err - } - if newMessageOpaque == nil { - return zeroValue, true, nil - } - if !changed { - return message, false, nil - } - newMessage, _ := newMessageOpaque.(T) // Safe to assert. - return newMessage, true, nil -} - func remapSlice[T any]( sourcePathsRemap *sourcePathsRemapTrie, sourcePath protoreflect.SourcePath, @@ -518,7 +501,7 @@ func remapSlice[T any]( isDirty = isDirty || changed if isDirty && newList == nil { if options.mutateInPlace { - newList = list[:toIndex:cap(list)] + newList = list[:toIndex] } else { newList = append(newList, list[:toIndex]...) } @@ -538,13 +521,14 @@ func remapSlice[T any]( fromIndex++ } if toIndex == 0 { + isDirty = true sourcePathsRemap.markDeleted(sourcePath) } if isDirty { if len(newList) == 0 { return nil, true, nil } - if options.mutateInPlace && newList != nil { + if options.mutateInPlace { // Zero out the remaining elements. for i := int(toIndex); i < len(list); i++ { list[i] = nil From 3c13e37a2329def59b577a63e11180c9ec95ef77 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 28 Mar 2025 10:58:28 +0100 Subject: [PATCH 79/80] Format testdata --- .../generate/testdata/v2/types/a/v1/a.proto | 14 +++++++------- .../generate/testdata/v2/types/b/v1/b.proto | 2 +- .../testdata/v2/types/pkg/v1/options.proto | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto b/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto index f0ac539061..f5466e41c7 100644 --- a/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/a/v1/a.proto @@ -2,16 +2,16 @@ syntax = "proto3"; package a.v1; -import "pkg/v1/options.proto"; import "b/v1/b.proto"; +import "pkg/v1/options.proto"; message Foo { - option (pkg.v1.message_foo).foo = "str"; - message Bar { - option (pkg.v1.message_bar).bar = "str"; - string bar = 1; - } - Bar nested_bar = 1; + option (pkg.v1.message_foo).foo = "str"; + message Bar { + option (pkg.v1.message_bar).bar = "str"; + string bar = 1; + } + Bar nested_bar = 1; b.v1.Baz baz = 2; } diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto b/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto index 75a12c13cc..2907baac9e 100644 --- a/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/b/v1/b.proto @@ -5,5 +5,5 @@ package b.v1; import "pkg/v1/options.proto"; message Baz { - option (pkg.v1.message_baz) = "str"; + option (pkg.v1.message_baz) = "str"; } diff --git a/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto b/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto index 18c8301071..61e186b7ef 100644 --- a/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto +++ b/private/buf/cmd/buf/command/generate/testdata/v2/types/pkg/v1/options.proto @@ -1,19 +1,19 @@ syntax = "proto3"; -import "google/protobuf/descriptor.proto"; - package pkg.v1; +import "google/protobuf/descriptor.proto"; + message OptionFoo { - string foo = 1; + string foo = 1; } message OptionBar { - string bar = 1; + string bar = 1; } extend google.protobuf.MessageOptions { - optional OptionFoo message_foo = 50000; - optional OptionBar message_bar = 50001; + optional OptionFoo message_foo = 50000; + optional OptionBar message_bar = 50001; optional string message_baz = 50002; } From 1c78fc96796f5c32e835cebef866678717371b3e Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Fri, 28 Mar 2025 10:58:42 +0100 Subject: [PATCH 80/80] Test consecutive fitlers --- .../bufpkg/bufimage/bufimageutil/bufimageutil.go | 4 ++-- .../bufimage/bufimageutil/bufimageutil_test.go | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go index 2e9cb9f1d6..3ad9b2eb9a 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil.go @@ -873,8 +873,8 @@ func (t *transitiveClosure) exploreCustomOptions( optionsByNumber := imageIndex.NameToOptions[optionsName] field, ok := optionsByNumber[int32(fd.Number())] if !ok { - err = fmt.Errorf("cannot find ext no %d on %s", fd.Number(), optionsName) - return false + // Option is unrecognized, ignore it. + return true } err = t.addElement(field, referrerFile, true, imageIndex, opts) return err == nil diff --git a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go index 6f8bbca33a..6ff80ef10b 100644 --- a/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go +++ b/private/bufpkg/bufimage/bufimageutil/bufimageutil_test.go @@ -463,6 +463,22 @@ func TestMutateInPlace(t *testing.T) { assert.Equal(t, filterLocationLen, len(aFileDescriptorProto.SourceCodeInfo.Location)) } +func TestConsecutiveFilters(t *testing.T) { + t.Parallel() + + ctx := context.Background() + _, image, err := getImage(ctx, slogtestext.NewLogger(t), "testdata/options") + require.NoError(t, err) + + t.Run("options", func(t *testing.T) { + t.Parallel() + filteredImage, err := FilterImage(image, WithIncludeTypes("pkg.Foo"), WithExcludeTypes("message_baz")) + require.NoError(t, err) + _, err = FilterImage(filteredImage, WithExcludeTypes("message_foo")) + require.NoError(t, err) + }) +} + func getImage(ctx context.Context, logger *slog.Logger, testdataDir string, options ...bufimage.BuildImageOption) (storage.ReadWriteBucket, bufimage.Image, error) { bucket, err := storageos.NewProvider().NewReadWriteBucket(testdataDir) if err != nil {