Skip to content

refactor: parse protos #343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kybxd wants to merge 1 commit into
base: master
Choose a base branch
from
Open

refactor: parse protos #343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/confgen/confgen_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func TestGenerator_GenWorkbook(t *testing.T) {
&options.ConfOption{
Input: &options.ConfInputOption{
ProtoPaths: []string{"../../test/functest/proto/default"},
ProtoFiles: []string{"../../test/functest/proto/default/*.proto"},
ProtoFiles: []string{"../../test/functest/proto/default/*.proto", "../../test/functest/proto/default/**/*.proto"},
Formats: []format.Format{
// format.Excel,
format.CSV,
Expand Down
4 changes: 2 additions & 2 deletions internal/protogen/protogen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,9 @@ func (gen *Generator) preprocess(useGeneratedProtos, delExisted bool) error {
}
}
// parse custom imported proto files
protoRegistryFiles, err := protoc.ParseProtos(
protoRegistryFiles, err := protoc.NewFiles(
gen.InputOpt.ProtoPaths,
protoFiles...)
protoFiles)
if err != nil {
return err
}
Expand Down
18 changes: 9 additions & 9 deletions internal/protogen/protogen_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,9 @@ func TestGenerator_GenWorkbook(t *testing.T) {
Input: &options.ProtoInputOption{
ProtoPaths: []string{outdir},
ProtoFiles: []string{
"common/base.proto",
"common/common.proto",
"common/union.proto",
filepath.Join(outdir, "common/base.proto"),
filepath.Join(outdir, "common/common.proto"),
filepath.Join(outdir, "common/union.proto"),
},
Formats: []format.Format{
format.YAML, format.CSV,
Expand All @@ -112,9 +112,9 @@ func TestGenerator_GenWorkbook(t *testing.T) {
Input: &options.ProtoInputOption{
ProtoPaths: []string{outdir},
ProtoFiles: []string{
"common/base.proto",
"common/common.proto",
"common/union.proto",
filepath.Join(outdir, "common/base.proto"),
filepath.Join(outdir, "common/common.proto"),
filepath.Join(outdir, "common/union.proto"),
},
Formats: []format.Format{
format.YAML, format.CSV,
Expand All @@ -141,9 +141,9 @@ func TestGenerator_GenWorkbook(t *testing.T) {
Input: &options.ProtoInputOption{
ProtoPaths: []string{outdir},
ProtoFiles: []string{
"common/base.proto",
"common/common.proto",
"common/union.proto",
filepath.Join(outdir, "common/base.proto"),
filepath.Join(outdir, "common/common.proto"),
filepath.Join(outdir, "common/union.proto"),
},
Formats: []format.Format{
format.YAML, format.CSV,
Expand Down
49 changes: 36 additions & 13 deletions internal/x/xproto/protoc/compiler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package protoc

import (
"context"
"io"
"io/fs"
"os"
"path/filepath"
"strings"

"github.com/bufbuild/protocompile"
"github.com/bufbuild/protocompile/linker"
Expand All @@ -17,13 +19,19 @@ import (
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/types/descriptorpb"

_ "github.com/tableauio/tableau/proto/tableaupb"
)

type fileDescriptorProtoMap map[string]*descriptorpb.FileDescriptorProto // file path -> file desc proto

// NewFiles creates a new protoregistry.Files from the provided proto paths, files, and excluded proto files.
func NewFiles(protoPaths []string, protoFiles []string, excludedProtoFiles ...string) (*protoregistry.Files, error) {
parsedExcludedProtoFiles := map[string]bool{}
cleanSlashProtoPaths := make([]string, len(protoPaths))
for i, protoPath := range protoPaths {
cleanSlashProtoPaths[i] = xfs.CleanSlashPath(protoPath)
}
parsedExcludedProtoFiles := make(map[string]bool)
for _, filename := range excludedProtoFiles {
matches, err := filepath.Glob(filename)
if err != nil {
Expand All @@ -34,7 +42,7 @@ func NewFiles(protoPaths []string, protoFiles []string, excludedProtoFiles ...st
parsedExcludedProtoFiles[cleanSlashPath] = true
}
}
var parsedProtoFiles []string
parsedProtoFiles := make(map[string]string) // full path -> rel path
for _, filename := range protoFiles {
matches, err := filepath.Glob(filename)
if err != nil {
Expand All @@ -43,27 +51,42 @@ func NewFiles(protoPaths []string, protoFiles []string, excludedProtoFiles ...st
for _, match := range matches {
cleanSlashPath := xfs.CleanSlashPath(match)
if !parsedExcludedProtoFiles[cleanSlashPath] {
for _, protoPath := range protoPaths {
cleanProtoPath := xfs.CleanSlashPath(protoPath) + "/"
cleanSlashPath = strings.TrimPrefix(cleanSlashPath, cleanProtoPath)
}
parsedProtoFiles = append(parsedProtoFiles, cleanSlashPath)
rel := rel(cleanSlashPath, cleanSlashProtoPaths)
parsedProtoFiles[cleanSlashPath] = rel
Comment on lines +54 to +55
Copy link
Member

@wenchy wenchy Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The excludedProtoFiles should be tested in unittest.

}
}
}
log.Debugf("proto files: %v", parsedProtoFiles)
return ParseProtos(protoPaths, parsedProtoFiles...)
return parseProtos(protoPaths, parsedProtoFiles)
}

// ParseProtos parses the proto paths and proto files to protoregistry.Files.
func ParseProtos(protoPaths []string, protoFiles ...string) (*protoregistry.Files, error) {
func rel(filename string, protoPaths []string) string {
for _, protoPath := range protoPaths {
if rel, err := filepath.Rel(protoPath, filename); err == nil {
return rel
}
}
return filename
}

// parseProtos parses the proto paths and proto files to protoregistry.Files.
func parseProtos(protoPaths []string, protoFilesMap map[string]string) (*protoregistry.Files, error) {
log.Debugf("proto paths: %v", protoPaths)
log.Debugf("proto files: %v", protoFiles)
log.Debugf("proto files: %v", protoFilesMap)
var protoFiles []string
for _, path := range protoFilesMap {
protoFiles = append(protoFiles, path)
}
compiler := protocompile.Compiler{
Resolver: protocompile.CompositeResolver{
protocompile.ResolverFunc(resolveGlobalFiles),
&protocompile.SourceResolver{
ImportPaths: protoPaths,
Accessor: func(path string) (io.ReadCloser, error) {
if _, ok := protoFilesMap[path]; !ok {
return nil, fs.ErrNotExist
}
return os.Open(path)
},
},
},
MaxParallelism: 1,
Expand Down
34 changes: 0 additions & 34 deletions internal/x/xproto/protoc/compiler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,37 +38,3 @@ func TestNewFiles(t *testing.T) {
})
}
}

func TestParseProtos(t *testing.T) {
Copy link
Member

@wenchy wenchy Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please improve it, but not deprecate it.

type args struct {
protoPaths []string
protoFiles []string
}
tests := []struct {
name string
args args
}{
// TODO: Add test cases.
{
name: "test1",
args: args{
protoPaths: []string{
"../../../../proto", // tableau
},
protoFiles: []string{
"tableau/protobuf/unittest/unittest.proto",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
files, err := ParseProtos(tt.args.protoPaths, tt.args.protoFiles...)
if err != nil {
t.Errorf("parseProtos() error = %v", err)
}
t.Logf("parsed proto files: %+v", files)
})
}
}

6 changes: 3 additions & 3 deletions internal/x/xproto/type_info_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,12 @@ func TestTypeInfos_Get(t *testing.T) {

func TestCloneWellknownTypes(t *testing.T) {
importPaths := []string{
"../../proto", // tableau
"../../../proto", // tableau
}
filenames := []string{
"tableau/protobuf/unittest/unittest.proto",
"../../../proto/tableau/protobuf/unittest/unittest.proto",
}
files, err := protoc.ParseProtos(importPaths, filenames...)
files, err := protoc.NewFiles(importPaths, filenames)
if err != nil {
t.Errorf("parseProtos() error = %v", err)
}
Expand Down
11 changes: 6 additions & 5 deletions options/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ type ProtoInputOption struct {
// The enums and messages in ProtoFiles can be used in Excel/CSV/XML/YAML
// as common types.
//
// NOTE: Glob patterns are supported, which can specify sets
// of filenames with wildcard characters.
//
// Default: nil.
ProtoFiles []string `yaml:"protoFiles"`

Expand All @@ -123,7 +126,7 @@ type ProtoInputOption struct {
//
// Default: nil.
Formats []format.Format `yaml:"formats"`

// Specify only these subdirs (relative to input dir) to be processed.
//
// Default: nil.
Expand Down Expand Up @@ -222,10 +225,8 @@ type ConfInputOption struct {

// The files to be parsed to generate configurations.
//
// NOTE:
// - Recognize "*.proto" pattern if not set (value is nil).
Copy link
Member

@wenchy wenchy Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, the default behavior is ok, and should not be removed. It's useful for simple tableauc job without config.yaml. For example, the https://tableauio.github.io/docs/prologue/quick-start/. If removed, it will not work well.

// - Glob patterns are supported, which can specify sets
// of filenames with wildcard characters.
// NOTE: Glob patterns are supported, which can specify sets
// of filenames with wildcard characters.
//
// Default: nil.
ProtoFiles []string `yaml:"protoFiles"`
Expand Down
6 changes: 3 additions & 3 deletions test/functest/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ func genProto(logLevel, logMode string) error {
Input: &options.ProtoInputOption{
ProtoPaths: []string{defaultOutdir},
ProtoFiles: []string{
"common/base.proto",
"common/common.proto",
"common/union.proto",
filepath.Join(defaultOutdir, "common/base.proto"),
Copy link
Member

@wenchy wenchy Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a breaking change, we should describe it in the PR title and description.

filepath.Join(defaultOutdir, "common/common.proto"),
filepath.Join(defaultOutdir, "common/union.proto"),
},
Formats: []format.Format{
// format.Excel,
Expand Down
Loading