refactor: change regex match to the glob path pattern (#73)

Signed-off-by: chlins <[email protected]>

Author: chlins

docs/getting-started.md

@@ -46,20 +46,14 @@ PRECISION bf16
 # Model quantization (string), such as awq, gptq, etc.
 QUANTIZATION awq
 
-# Specify model configuration file.
+# Specify model configuration file, support glob path pattern.
 CONFIG config.json
 
-# Specify model configuration file.
+# Specify model configuration file, support glob path pattern.
 CONFIG generation_config.json
 
-# Model weight.
-MODEL \.safetensors$
-
-# Model code.
-CODE \.py$
-
-# Model dataset.
-DATASET \.csv$
+# Model weight, support glob path pattern.
+MODEL *.safetensors
 ```
 
 Then run the following command to build the model artifact:

pkg/backend/build.go

@@ -19,13 +19,12 @@ package backend
 import (
 	"context"
 	"fmt"
-	"os"
-	"path/filepath"
 
 	"github.com/CloudNativeAI/modctl/pkg/backend/build"
 	"github.com/CloudNativeAI/modctl/pkg/backend/processor"
 	"github.com/CloudNativeAI/modctl/pkg/modelfile"
 
+	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
 	humanize "github.com/dustin/go-humanize"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
@@ -45,7 +44,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 
 	repo, tag := ref.Repository(), ref.Tag()
 	layers := []ocispec.Descriptor{}
-	layerDescs, err := b.process(ctx, workDir, repo, getProcessors(modelfile)...)
+	layerDescs, err := b.process(ctx, workDir, repo, b.getProcessors(modelfile)...)
 	if err != nil {
 		return fmt.Errorf("failed to process files: %w", err)
 	}
@@ -70,64 +69,41 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 	return nil
 }
 
-func defaultProcessors() []processor.Processor {
+func (b *backend) defaultProcessors() []processor.Processor {
 	return []processor.Processor{
-		processor.NewLicenseProcessor(),
-		processor.NewReadmeProcessor(),
+		// by default process the readme and license file.
+		processor.NewReadmeProcessor(b.store, modelspec.MediaTypeModelDoc, []string{"README.md", "README"}),
+		processor.NewLicenseProcessor(b.store, modelspec.MediaTypeModelDoc, []string{"LICENSE.txt", "LICENSE"}),
 	}
 }
 
-func getProcessors(modelfile modelfile.Modelfile) []processor.Processor {
-	processors := defaultProcessors()
+func (b *backend) getProcessors(modelfile modelfile.Modelfile) []processor.Processor {
+	processors := b.defaultProcessors()
 
 	if configs := modelfile.GetConfigs(); len(configs) > 0 {
-		processors = append(processors, processor.NewModelConfigProcessor(configs))
+		processors = append(processors, processor.NewModelConfigProcessor(b.store, modelspec.MediaTypeModelWeightConfig, configs))
 	}
 
 	if models := modelfile.GetModels(); len(models) > 0 {
-		processors = append(processors, processor.NewModelProcessor(models))
+		processors = append(processors, processor.NewModelProcessor(b.store, modelspec.MediaTypeModelWeight, models))
 	}
 
 	return processors
 }
 
 // process walks the user work directory and process the identified files.
 func (b *backend) process(ctx context.Context, workDir string, repo string, processors ...processor.Processor) ([]ocispec.Descriptor, error) {
-	layers := []ocispec.Descriptor{}
-	// walk the user work directory and handle the default identified files.
-	if err := filepath.Walk(workDir, func(path string, info os.FileInfo, err error) error {
+	descriptors := []ocispec.Descriptor{}
+	for _, p := range processors {
+		descs, err := p.Process(ctx, workDir, repo)
 		if err != nil {
-			return fmt.Errorf("failed to walk directory: %w", err)
-		}
-		// skip directories.
-		if info.IsDir() {
-			return nil
-		}
-		// get absolute path.
-		path, err = filepath.Abs(path)
-		if err != nil {
-			return fmt.Errorf("failed to get absolute path: %w", err)
-		}
-		// fan-in file processors.
-		for _, p := range processors {
-			// process the file if it can be recognized.
-			if p.Identify(ctx, path, info) {
-				desc, err := p.Process(ctx, b.store, repo, path, workDir)
-				if err != nil {
-					return fmt.Errorf("failed to process file: %w", err)
-				}
-
-				fmt.Printf("%-15s => %s (%s)\n", "Built blob", desc.Digest, humanize.IBytes(uint64(desc.Size)))
-				layers = append(layers, desc)
-			}
+			return nil, err
 		}
 
-		return nil
-	}); err != nil {
-		return nil, err
+		descriptors = append(descriptors, descs...)
 	}
 
-	return layers, nil
+	return descriptors, nil
 }
 
 // manifestAnnotation returns the annotations for the manifest.

pkg/backend/build/build.go

@@ -36,7 +36,7 @@ import (
 )
 
 // BuildLayer converts the file to the image blob and push it to the storage.
-func BuildLayer(ctx context.Context, store storage.Storage, mediaType, repo, path, workDir string) (ocispec.Descriptor, error) {
+func BuildLayer(ctx context.Context, store storage.Storage, mediaType, workDir, repo, path string) (ocispec.Descriptor, error) {
 	reader, err := archiver.Tar(path)
 	if err != nil {
 		return ocispec.Descriptor{}, fmt.Errorf("failed to tar file: %w", err)

pkg/backend/build_test.go

@@ -29,11 +29,12 @@ func TestGetProcessors(t *testing.T) {
 	modelfile.On("GetConfigs").Return([]string{"config1", "config2"})
 	modelfile.On("GetModels").Return([]string{"model1", "model2"})
 
-	processors := getProcessors(modelfile)
+	b := &backend{}
+	processors := b.getProcessors(modelfile)
 
 	assert.Len(t, processors, 4)
-	assert.Equal(t, "license", processors[0].Name())
-	assert.Equal(t, "readme", processors[1].Name())
+	assert.Equal(t, "readme", processors[0].Name())
+	assert.Equal(t, "license", processors[1].Name())
 	assert.Equal(t, "model_config", processors[2].Name())
 	assert.Equal(t, "model", processors[3].Name())
 }

pkg/backend/processor/base.go

@@ -0,0 +1,69 @@
+/*
+ *     Copyright 2025 The CNAI Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package processor
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/CloudNativeAI/modctl/pkg/backend/build"
+	"github.com/CloudNativeAI/modctl/pkg/storage"
+
+	humanize "github.com/dustin/go-humanize"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+type base struct {
+	// store is the underlying storage backend.
+	store storage.Storage
+	// mediaType is the media type of the processed content.
+	mediaType string
+	// patterns is the list of patterns to match.
+	patterns []string
+}
+
+// Process implements the Processor interface, which can be reused by other processors.
+func (b *base) Process(ctx context.Context, workDir, repo string) ([]ocispec.Descriptor, error) {
+	absWorkDir, err := filepath.Abs(workDir)
+	if err != nil {
+		return nil, err
+	}
+
+	var matchedPaths []string
+	for _, pattern := range b.patterns {
+		matches, err := filepath.Glob(filepath.Join(absWorkDir, pattern))
+		if err != nil {
+			return nil, err
+		}
+
+		matchedPaths = append(matchedPaths, matches...)
+	}
+
+	var descriptors []ocispec.Descriptor
+	for _, path := range matchedPaths {
+		desc, err := build.BuildLayer(ctx, b.store, b.mediaType, workDir, repo, path)
+		if err != nil {
+			return nil, err
+		}
+
+		fmt.Printf("%-15s => %s (%s)\n", "Built blob", desc.Digest, humanize.IBytes(uint64(desc.Size)))
+		descriptors = append(descriptors, desc)
+	}
+
+	return descriptors, nil
+}

pkg/backend/processor/license.go

@@ -18,36 +18,31 @@ package processor
 
 import (
 	"context"
-	"os"
 
-	"github.com/CloudNativeAI/modctl/pkg/backend/build"
 	"github.com/CloudNativeAI/modctl/pkg/storage"
-	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
-
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // NewLicenseProcessor creates a new LICENSE processor.
-func NewLicenseProcessor() Processor {
-	return &licenseProcessor{}
+func NewLicenseProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+	return &licenseProcessor{
+		base: &base{
+			store:     store,
+			mediaType: mediaType,
+			patterns:  patterns,
+		},
+	}
 }
 
 // licenseProcessor is the processor to process the LICENSE file.
-type licenseProcessor struct{}
+type licenseProcessor struct {
+	base *base
+}
 
 func (p *licenseProcessor) Name() string {
 	return "license"
 }
 
-func (p *licenseProcessor) Identify(_ context.Context, path string, info os.FileInfo) bool {
-	return info.Name() == "LICENSE" || info.Name() == "LICENSE.txt"
-}
-
-func (p *licenseProcessor) Process(ctx context.Context, store storage.Storage, repo, path, workDir string) (ocispec.Descriptor, error) {
-	desc, err := build.BuildLayer(ctx, store, modelspec.MediaTypeModelDoc, repo, path, workDir)
-	if err != nil {
-		return ocispec.Descriptor{}, err
-	}
-
-	return desc, nil
+func (p *licenseProcessor) Process(ctx context.Context, workDir, repo string) ([]ocispec.Descriptor, error) {
+	return p.base.Process(ctx, workDir, repo)
 }

pkg/backend/processor/license_test.go

@@ -18,54 +18,52 @@ package processor
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"testing"
-	"testing/fstest"
 
 	"github.com/CloudNativeAI/modctl/test/mocks/storage"
 	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
 )
 
-func TestLicenseProcessor_Name(t *testing.T) {
-	p := NewLicenseProcessor()
-	assert.Equal(t, "license", p.Name())
+type licenseProcessorSuite struct {
+	suite.Suite
+	mockStore *storage.Storage
+	processor Processor
+	workDir   string
 }
 
-func TestLicenseProcessor_Identify(t *testing.T) {
-	p := NewLicenseProcessor()
-	mockFS := fstest.MapFS{
-		"LICENSE":     &fstest.MapFile{},
-		"LICENSE.txt": &fstest.MapFile{},
-		"README.md":   &fstest.MapFile{},
+func (s *licenseProcessorSuite) SetupTest() {
+	s.mockStore = &storage.Storage{}
+	s.processor = NewLicenseProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"})
+	// generate test files for prorcess.
+	s.workDir = s.Suite.T().TempDir()
+	if err := os.WriteFile(filepath.Join(s.workDir, "LICENSE"), []byte(""), 0644); err != nil {
+		s.Suite.T().Fatal(err)
 	}
-	info, err := mockFS.Stat("LICENSE")
-	assert.NoError(t, err)
-	assert.True(t, p.Identify(context.Background(), "LICENSE", info))
-
-	info, err = mockFS.Stat("LICENSE.txt")
-	assert.NoError(t, err)
-	assert.True(t, p.Identify(context.Background(), "LICENSE.txt", info))
+}
 
-	info, err = mockFS.Stat("README.md")
-	assert.NoError(t, err)
-	assert.False(t, p.Identify(context.Background(), "README.md", info))
+func (s *licenseProcessorSuite) TestName() {
+	assert.Equal(s.Suite.T(), "license", s.processor.Name())
 }
 
-func TestLicenseProcessor_Process(t *testing.T) {
-	p := NewLicenseProcessor()
+func (s *licenseProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	mockStore := &storage.Storage{}
 	repo := "test-repo"
-	path := "/tmp/LICENSE"
+	s.mockStore.On("PushBlob", ctx, repo, mock.Anything).Return("sha256:1234567890abcdef", int64(1024), nil)
 
-	mockStore.On("PushBlob", ctx, repo, mock.Anything).Return("sha256:1234567890abcdef", int64(1024), nil)
+	desc, err := s.processor.Process(ctx, s.workDir, repo)
+	assert.NoError(s.Suite.T(), err)
+	assert.NotNil(s.Suite.T(), desc)
+	assert.Equal(s.Suite.T(), "sha256:1234567890abcdef", desc[0].Digest.String())
+	assert.Equal(s.Suite.T(), int64(1024), desc[0].Size)
+	assert.Equal(s.Suite.T(), "LICENSE", desc[0].Annotations[modelspec.AnnotationFilepath])
+}
 
-	desc, err := p.Process(ctx, mockStore, repo, path, "/tmp")
-	assert.NoError(t, err)
-	assert.NotNil(t, desc)
-	assert.Equal(t, "sha256:1234567890abcdef", desc.Digest.String())
-	assert.Equal(t, int64(1024), desc.Size)
-	assert.Equal(t, "LICENSE", desc.Annotations[modelspec.AnnotationFilepath])
+func TestLicenseProcessorSuite(t *testing.T) {
+	suite.Run(t, new(licenseProcessorSuite))
 }