feat: introduce the DOC instruction (#92)

Signed-off-by: chlins <[email protected]>

Author: chlins

docs/getting-started.md

@@ -58,6 +58,9 @@ MODEL *.safetensors
 # Specify code, support glob path pattern.
 CODE *.py
 
+# Specify documentation, support glob path pattern.
+DOC *.md
+
 ```
 
 Then run the following command to build the model artifact:
@@ -110,8 +113,8 @@ Delete the model artifact in the local storage:
 $ modctl rm registry.com/models/llama3:v1.0.0
 ```
 
-Finally, you can use `purge` command to to remove all unnecessary blobs to free up the storage space:
+Finally, you can use `prune` command to remove all unnecessary blobs to free up the storage space:
 
 ```shell
-$ modctl purge
+$ modctl prune
 ```

pkg/backend/build.go

@@ -73,16 +73,8 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 	return nil
 }
 
-func (b *backend) defaultProcessors() []processor.Processor {
-	return []processor.Processor{
-		// by default process the readme and license file.
-		processor.NewReadmeProcessor(b.store, modelspec.MediaTypeModelDoc, []string{"README.md", "README"}),
-		processor.NewLicenseProcessor(b.store, modelspec.MediaTypeModelDoc, []string{"LICENSE.txt", "LICENSE"}),
-	}
-}
-
 func (b *backend) getProcessors(modelfile modelfile.Modelfile) []processor.Processor {
-	processors := b.defaultProcessors()
+	processors := []processor.Processor{}
 
 	if configs := modelfile.GetConfigs(); len(configs) > 0 {
 		processors = append(processors, processor.NewModelConfigProcessor(b.store, modelspec.MediaTypeModelWeightConfig, configs))
@@ -96,6 +88,10 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile) []processor.Proce
 		processors = append(processors, processor.NewCodeProcessor(b.store, modelspec.MediaTypeModelCode, codes))
 	}
 
+	if docs := modelfile.GetDocs(); len(docs) > 0 {
+		processors = append(processors, processor.NewDocProcessor(b.store, modelspec.MediaTypeModelDoc, docs))
+	}
+
 	return processors
 }
 

pkg/backend/build_test.go

@@ -29,14 +29,14 @@ func TestGetProcessors(t *testing.T) {
 	modelfile.On("GetConfigs").Return([]string{"config1", "config2"})
 	modelfile.On("GetModels").Return([]string{"model1", "model2"})
 	modelfile.On("GetCodes").Return([]string{"1.py", "2.py"})
+	modelfile.On("GetDocs").Return([]string{"doc1", "doc2"})
 
 	b := &backend{}
 	processors := b.getProcessors(modelfile)
 
-	assert.Len(t, processors, 5)
-	assert.Equal(t, "readme", processors[0].Name())
-	assert.Equal(t, "license", processors[1].Name())
-	assert.Equal(t, "model_config", processors[2].Name())
-	assert.Equal(t, "model", processors[3].Name())
-	assert.Equal(t, "code", processors[4].Name())
+	assert.Len(t, processors, 4)
+	assert.Equal(t, "model_config", processors[0].Name())
+	assert.Equal(t, "model", processors[1].Name())
+	assert.Equal(t, "code", processors[2].Name())
+	assert.Equal(t, "doc", processors[3].Name())
 }

pkg/backend/processor/base.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"sort"
 
 	"github.com/CloudNativeAI/modctl/pkg/backend/build"
 	"github.com/CloudNativeAI/modctl/pkg/storage"
@@ -54,6 +55,8 @@ func (b *base) Process(ctx context.Context, workDir, repo string) ([]ocispec.Des
 		matchedPaths = append(matchedPaths, matches...)
 	}
 
+	sort.Strings(matchedPaths)
+
 	var descriptors []ocispec.Descriptor
 	for _, path := range matchedPaths {
 		desc, err := build.BuildLayer(ctx, b.store, b.mediaType, workDir, repo, path)

pkg/backend/processor/readme.go pkg/backend/processor/doc.gopkg/backend/processor/readme.go renamed to pkg/backend/processor/doc.go

@@ -20,13 +20,12 @@ import (
 	"context"
 
 	"github.com/CloudNativeAI/modctl/pkg/storage"
-
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// NewReadmeProcessor creates a new README processor.
-func NewReadmeProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
-	return &readmeProcessor{
+// NewDocProcessor creates a new doc processor.
+func NewDocProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+	return &docProcessor{
 		base: &base{
 			store:     store,
 			mediaType: mediaType,
@@ -35,15 +34,15 @@ func NewReadmeProcessor(store storage.Storage, mediaType string, patterns []stri
 	}
 }
 
-// readmeProcessor is the processor to process the README file.
-type readmeProcessor struct {
+// docProcessor is the processor to process the doc file.
+type docProcessor struct {
 	base *base
 }
 
-func (p *readmeProcessor) Name() string {
-	return "readme"
+func (p *docProcessor) Name() string {
+	return "doc"
 }
 
-func (p *readmeProcessor) Process(ctx context.Context, workDir, repo string) ([]ocispec.Descriptor, error) {
+func (p *docProcessor) Process(ctx context.Context, workDir, repo string) ([]ocispec.Descriptor, error) {
 	return p.base.Process(ctx, workDir, repo)
 }

pkg/backend/processor/license_test.go pkg/backend/processor/doc_test.gopkg/backend/processor/license_test.go renamed to pkg/backend/processor/doc_test.go

@@ -30,28 +30,28 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
-type licenseProcessorSuite struct {
+type docProcessorSuite struct {
 	suite.Suite
 	mockStore *storage.Storage
 	processor Processor
 	workDir   string
 }
 
-func (s *licenseProcessorSuite) SetupTest() {
+func (s *docProcessorSuite) SetupTest() {
 	s.mockStore = &storage.Storage{}
-	s.processor = NewLicenseProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"})
+	s.processor = NewDocProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"})
 	// generate test files for prorcess.
 	s.workDir = s.Suite.T().TempDir()
 	if err := os.WriteFile(filepath.Join(s.workDir, "LICENSE"), []byte(""), 0644); err != nil {
 		s.Suite.T().Fatal(err)
 	}
 }
 
-func (s *licenseProcessorSuite) TestName() {
-	assert.Equal(s.Suite.T(), "license", s.processor.Name())
+func (s *docProcessorSuite) TestName() {
+	assert.Equal(s.Suite.T(), "doc", s.processor.Name())
 }
 
-func (s *licenseProcessorSuite) TestProcess() {
+func (s *docProcessorSuite) TestProcess() {
 	ctx := context.Background()
 	repo := "test-repo"
 	s.mockStore.On("PushBlob", ctx, repo, mock.Anything).Return("sha256:1234567890abcdef", int64(1024), nil)
@@ -64,6 +64,6 @@ func (s *licenseProcessorSuite) TestProcess() {
 	assert.Equal(s.Suite.T(), "LICENSE", desc[0].Annotations[modelspec.AnnotationFilepath])
 }
 
-func TestLicenseProcessorSuite(t *testing.T) {
-	suite.Run(t, new(licenseProcessorSuite))
+func TestDocProcessorSuite(t *testing.T) {
+	suite.Run(t, new(docProcessorSuite))
 }

pkg/backend/processor/license.go

@@ -45,6 +45,13 @@ const (
 	// package, and each dataset file will be a layer.
 	DATASET = "dataset"
 
+	// DOC is the command to set the documentation file path. The value of this commands
+	// is the glob of the documentation file path to match the documentation file name.
+	// The DOC command can be used multiple times in a modelfile, it will scan
+	// the documentation file path by the glob and copy each documentation file to the artifact
+	// package, and each documentation file will be a layer.
+	DOC = "doc"
+
 	// NAME is the command to set the model name, such as llama3-8b-instruct, gpt2-xl,
 	// qwen2-vl-72b-instruct, etc.
 	NAME = "name"
@@ -75,6 +82,7 @@ var Commands = []string{
 	MODEL,
 	CODE,
 	DATASET,
+	DOC,
 	NAME,
 	ARCH,
 	FAMILY,