feat: support modelfile generate with exclude patterns (modelpack#332)

* feat: enhance modelfile generate with exclude patterns

Signed-off-by: qmy <qiumuyang.qmy@antgroup.com>

* docs(getting-started): update modelfile generate usage

Signed-off-by: qmy <qiumuyang.qmy@antgroup.com>

---------

Signed-off-by: qmy <qiumuyang.qmy@antgroup.com>
Co-authored-by: qmy <qiumuyang.qmy@antgroup.com>

Author: qiumuyang

cmd/modelfile/generate.go

@@ -64,6 +64,7 @@ func init() {
 	flags.StringVarP(&generateConfig.Output, "output", "O", ".", "specify the output path of modelfilem, must be a directory")
 	flags.BoolVar(&generateConfig.IgnoreUnrecognizedFileTypes, "ignore-unrecognized-file-types", false, "ignore the unrecognized file types in the workspace")
 	flags.BoolVar(&generateConfig.Overwrite, "overwrite", false, "overwrite the existing modelfile")
+	flags.StringArrayVar(&generateConfig.ExcludePatterns, "exclude", []string{}, "specify glob patterns to exclude files/directories (e.g. *.log, checkpoints/*)")
 
 	// Mark the ignore-unrecognized-file-types flag as deprecated and hidden
 	flags.MarkDeprecated("ignore-unrecognized-file-types", "this flag will be removed in the next release")

docs/getting-started.md

@@ -31,6 +31,14 @@ directory(workspace).
 $ modctl modelfile generate .
 ```
 
+If you want to exclude specific files from the model artifact (such as checkpoint directories),
+you can use the `--exclude` option to specify the file path glob pattern.
+Note that only basic glob syntax (`*`, `?`, `[]`) is supported; advanced features like `**` for recursive matching are not.
+
+```shell
+$ modctl modelfile generate . --exclude 'checkpoint-*'
+```
+
 ### Build
 
 Build the model artifact you need to prepare a Modelfile describe your expected layout of the model artifact in your model repo.

pkg/config/modelfile/modelfile.go

@@ -39,6 +39,7 @@ type GenerateConfig struct {
 	ParamSize                   string
 	Precision                   string
 	Quantization                string
+	ExcludePatterns             []string
 }
 
 func NewGenerateConfig() *GenerateConfig {
@@ -55,6 +56,7 @@ func NewGenerateConfig() *GenerateConfig {
 		ParamSize:                   "",
 		Precision:                   "",
 		Quantization:                "",
+		ExcludePatterns:             []string{},
 	}
 }
 

pkg/modelfile/modelfile.go

@@ -209,7 +209,7 @@ func NewModelfileByWorkspace(workspace string, config *configmodelfile.GenerateC
 		return nil, err
 	}
 
-	if err := mf.generateByWorkspace(); err != nil {
+	if err := mf.generateByWorkspace(config); err != nil {
 		return nil, err
 	}
 
@@ -252,11 +252,17 @@ func (mf *modelfile) validateWorkspace() error {
 }
 
 // generateByWorkspace generates the modelfile by the workspace's files.
-func (mf *modelfile) generateByWorkspace() error {
+func (mf *modelfile) generateByWorkspace(config *configmodelfile.GenerateConfig) error {
 	// Initialize counters for workspace limits validation
 	var fileCount int
 	var totalSize int64
 
+	// Initialize exclude patterns
+	filter, err := NewPathFilter(config.ExcludePatterns...)
+	if err != nil {
+		return err
+	}
+
 	// Walk the path and get the files.
 	if err := filepath.Walk(mf.workspace, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -265,8 +271,14 @@ func (mf *modelfile) generateByWorkspace() error {
 
 		filename := info.Name()
 
-		// Skip hidden and skippable files/directories.
-		if isSkippable(filename) {
+		// Get relative path from the base directory.
+		relPath, err := filepath.Rel(mf.workspace, path)
+		if err != nil {
+			return err
+		}
+
+		// Skip hidden, skippable, and excluded files/directories.
+		if isSkippable(filename) || filter.Match(relPath) {
 			if info.IsDir() {
 				return filepath.SkipDir
 			}
@@ -298,12 +310,6 @@ func (mf *modelfile) generateByWorkspace() error {
 			return fmt.Errorf("workspace exceeds maximum total size limit of %d bytes (%s)", MaxTotalWorkspaceSize, formatBytes(MaxTotalWorkspaceSize))
 		}
 
-		// Get relative path from the base directory.
-		relPath, err := filepath.Rel(mf.workspace, path)
-		if err != nil {
-			return err
-		}
-
 		switch {
 		case IsFileType(filename, ConfigFilePatterns):
 			mf.config.Add(relPath)

pkg/modelfile/path_filter.go

@@ -0,0 +1,44 @@
+package modelfile
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type PathFilter struct {
+	patterns []string
+}
+
+func NewPathFilter(patterns ...string) (*PathFilter, error) {
+	var cleaned []string
+	for _, p := range patterns {
+		// validate the pattern
+		if _, err := filepath.Match(p, ""); err != nil {
+			return nil, fmt.Errorf("invalid exclude pattern: %q", p)
+		}
+		// since filepath.Walk never returns a path with trailing separator, we need to remove separator from patterns
+		cleaned = append(cleaned, strings.TrimRight(p, string(filepath.Separator)))
+	}
+	return &PathFilter{patterns: cleaned}, nil
+}
+
+func (pf *PathFilter) Match(path string) bool {
+	if len(pf.patterns) == 0 {
+		return false
+	}
+
+	for _, pattern := range pf.patterns {
+		matched, err := filepath.Match(pattern, path)
+		if err != nil {
+			// The only possible returned error is ErrBadPattern
+			// which we checked when creating the filter
+			return false
+		}
+		if matched {
+			return true
+		}
+	}
+
+	return false
+}

pkg/modelfile/path_filter_test.go

@@ -0,0 +1,162 @@
+package modelfile
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewPathFilter(t *testing.T) {
+	testcases := []struct {
+		name        string
+		input       []string
+		expected    []string
+		expectError bool
+		errorMsg    string
+	}{
+		{
+			name:     "normal patterns",
+			input:    []string{"*.log", "checkpoint*/"},
+			expected: []string{"*.log", "checkpoint*"},
+		},
+		{
+			name:        "invalid pattern",
+			input:       []string{"*.log", "[invalid"},
+			expectError: true,
+			errorMsg:    `invalid exclude pattern: "[invalid"`,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			filter, err := NewPathFilter(tc.input...)
+
+			if tc.expectError {
+				require.Error(t, err, "Expected an error for input: %q", tc.input)
+				assert.Contains(t, err.Error(), tc.errorMsg)
+				assert.Nil(t, filter)
+				return
+			}
+
+			require.NoError(t, err, "Did not expect an error for input: %q", tc.input)
+			require.NotNil(t, filter)
+			assert.Equal(t, tc.expected, filter.patterns)
+		})
+	}
+}
+
+func TestPathFilter_Matches(t *testing.T) {
+	testcases := []struct {
+		filterName string
+		patterns   []string
+		tests      []struct {
+			desc     string
+			path     string
+			expected bool
+		}
+	}{
+		{
+			filterName: "Empty_Filter",
+			patterns:   []string{},
+			tests: []struct {
+				desc     string
+				path     string
+				expected bool
+			}{
+				{"any file", "any/path/file.txt", false},
+				{"root file", "main.go", false},
+				{"empty path", "", false},
+			},
+		},
+		{
+			filterName: "Single_Asterisk_Filter",
+			patterns:   []string{"*.log"},
+			tests: []struct {
+				desc     string
+				path     string
+				expected bool
+			}{
+				{"matches a simple log file", "debug.log", true},
+				{"matches a hidden log file", ".config.log", true},
+				{"does not match if not at end", "debug.log.old", false},
+				{"does not match different extension", "main.go", false},
+				{"does not cross path separator", "logs/debug.log", false},
+			},
+		},
+		{
+			filterName: "Directory_Wildcard_Filter",
+			patterns:   []string{"build/*"},
+			tests: []struct {
+				desc     string
+				path     string
+				expected bool
+			}{
+				{"matches file directly inside", "build/app", true},
+				{"matches hidden file inside", "build/.config", true},
+				{"does not match the directory itself", "build", false},
+				{"does not match nested files", "build/assets/icon.png", false},
+			},
+		},
+		{
+			// Since filepath.Match does not support **, the behavior is the same as Directory_Wildcard_Filter
+			filterName: "Directory_Double_Asterisk_Filter",
+			patterns:   []string{"build/**"},
+			tests: []struct {
+				desc     string
+				path     string
+				expected bool
+			}{
+				{"matches file directly inside", "build/app", true},
+				{"matches hidden file inside", "build/.config", true},
+				{"does not match the directory itself", "build", false},
+				{"does not match nested files", "build/assets/icon.png", false},
+			},
+		},
+		{
+			filterName: "Directory_Filter",
+			patterns:   []string{"checkpoint/"},
+			tests: []struct {
+				desc     string
+				path     string
+				expected bool
+			}{
+				{"matches the directory itself", "checkpoint", true},
+				{"match file inside", "checkpoint/file.py", false},
+			},
+		},
+		{
+			filterName: "Complex_Filter_With_Multiple_Patterns",
+			patterns:   []string{"*.tmp", ".git*", "build/"},
+			tests: []struct {
+				desc     string
+				path     string
+				expected bool
+			}{
+				{"matches a .tmp file", "temp.tmp", true},
+				{"matches .git directory", ".git", true},
+				{"matches .gitignore file", ".gitignore", true},
+				{"matches build directory exactly", "build", true},
+				{"does not cross separator", "data/cache.tmp", false},
+				{"does not match file inside build/", "build/app.js", false},
+				{"does not match src file", "src/main.go", false},
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.filterName, func(t *testing.T) {
+			filter, err := NewPathFilter(tc.patterns...)
+			require.NoError(t, err, "Filter creation with patterns %q failed", tc.patterns)
+			require.NotNil(t, filter)
+
+			for _, test := range tc.tests {
+				t.Run(test.desc, func(t *testing.T) {
+					result := filter.Match(test.path)
+					assert.Equal(t, test.expected, result, fmt.Sprintf("Path: %q", test.path))
+				})
+			}
+		})
+	}
+}