feat: support build the model artifact by raw file format (#188)

Signed-off-by: chlins <chlins.zhang@gmail.com>

Author: chlins

cmd/build.go

@@ -52,15 +52,17 @@ var buildCmd = &cobra.Command{
 func init() {
 	flags := buildCmd.Flags()
 	flags.IntVarP(&buildConfig.Concurrency, "concurrency", "c", buildConfig.Concurrency, "specify the number of concurrent build operations")
-	flags.StringVarP(&buildConfig.Target, "target", "t", "", "target model artifact name")
-	flags.StringVarP(&buildConfig.Modelfile, "modelfile", "f", "Modelfile", "model file path")
+	flags.StringVarP(&buildConfig.Target, "target", "t", buildConfig.Target, "target model artifact name")
+	flags.StringVarP(&buildConfig.Modelfile, "modelfile", "f", buildConfig.Modelfile, "model file path")
 	flags.BoolVarP(&buildConfig.OutputRemote, "output-remote", "", false, "turning on this flag will output model artifact to remote registry directly")
 	flags.BoolVarP(&buildConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
 	flags.BoolVarP(&buildConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
 	flags.BoolVar(&buildConfig.Nydusify, "nydusify", false, "[EXPERIMENTAL] nydusify the model artifact")
 	flags.MarkHidden("nydusify")
 	flags.StringVar(&buildConfig.SourceURL, "source-url", "", "source URL")
 	flags.StringVar(&buildConfig.SourceRevision, "source-revision", "", "source revision")
+	// TODO: set the raw flag to true by default in future.
+	flags.BoolVar(&buildConfig.Raw, "raw", false, "turning on this flag will build model artifact layers in raw format")
 
 	if err := viper.BindPFlags(flags); err != nil {
 		panic(fmt.Errorf("bind cache list flags to viper: %w", err))

go.mod

@@ -3,7 +3,7 @@ module github.com/CloudNativeAI/modctl
 go 1.24.1
 
 require (
-	github.com/CloudNativeAI/model-spec v0.0.3
+	github.com/CloudNativeAI/model-spec v0.0.5
 	github.com/antgroup/hugescm v0.18.0
 	github.com/avast/retry-go/v4 v4.6.1
 	github.com/briandowns/spinner v1.23.2

go.sum

@@ -2,8 +2,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/CloudNativeAI/model-spec v0.0.3 h1:5mvgFQ+3pyupzxYjtV5XAeg9zRe6+46pLPBFNHlOrqE=
-github.com/CloudNativeAI/model-spec v0.0.3/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk=
+github.com/CloudNativeAI/model-spec v0.0.5 h1:gVwvpiVJgGZynvIi+xy0Hp/zh+g1EuWa6x6Czm4P0uA=
+github.com/CloudNativeAI/model-spec v0.0.5/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=

pkg/archiver/archiver.go

@@ -164,6 +164,14 @@ func Untar(reader io.Reader, destPath string) error {
 			if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
 				return fmt.Errorf("failed to create directory %s: %w", targetPath, err)
 			}
+			// Set correct permissions for the directory.
+			if err := os.Chmod(targetPath, os.FileMode(header.Mode)); err != nil {
+				return fmt.Errorf("failed to set directory permissions %s: %w", targetPath, err)
+			}
+			// Set modification time for the directory.
+			if err := os.Chtimes(targetPath, header.ModTime, header.ModTime); err != nil {
+				return fmt.Errorf("failed to set directory mtime %s: %w", targetPath, err)
+			}
 
 		case tar.TypeReg:
 			file, err := os.OpenFile(
@@ -181,13 +189,13 @@ func Untar(reader io.Reader, destPath string) error {
 			}
 			file.Close()
 
-		case tar.TypeSymlink:
-			if isRel(header.Linkname, destPath) && isRel(header.Name, destPath) {
-				if err := os.Symlink(header.Linkname, targetPath); err != nil {
-					return fmt.Errorf("failed to create symlink %s -> %s: %w", targetPath, header.Linkname, err)
-				}
-			} else {
-				return fmt.Errorf("symlink %s -> %s points outside of destination directory", targetPath, header.Linkname)
+			// Set correct permissions for the directory.
+			if err := os.Chmod(targetPath, os.FileMode(header.Mode)); err != nil {
+				return fmt.Errorf("failed to set directory permissions %s: %w", targetPath, err)
+			}
+			// Set modification time for the file.
+			if err := os.Chtimes(targetPath, header.ModTime, header.ModTime); err != nil {
+				return fmt.Errorf("failed to set file mtime %s: %w", targetPath, err)
 			}
 
 		default:
@@ -198,18 +206,3 @@ func Untar(reader io.Reader, destPath string) error {
 
 	return nil
 }
-
-// isRel checks if the candidate path is within the target directory after resolving symbolic links.
-func isRel(candidate, target string) bool {
-	if filepath.IsAbs(candidate) {
-		return false
-	}
-
-	realpath, err := filepath.EvalSymlinks(filepath.Join(target, candidate))
-	if err != nil {
-		return false
-	}
-
-	relpath, err := filepath.Rel(target, realpath)
-	return err == nil && !strings.HasPrefix(filepath.Clean(relpath), "..")
-}

pkg/backend/build.go

@@ -90,7 +90,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 	defer pb.Stop()
 
 	layers := []ocispec.Descriptor{}
-	layerDescs, err := b.process(ctx, builder, workDir, pb, cfg, b.getProcessors(modelfile)...)
+	layerDescs, err := b.process(ctx, builder, workDir, pb, cfg, b.getProcessors(modelfile, cfg)...)
 	if err != nil {
 		return fmt.Errorf("failed to process files: %w", err)
 	}
@@ -154,23 +154,39 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 	return nil
 }
 
-func (b *backend) getProcessors(modelfile modelfile.Modelfile) []processor.Processor {
+func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build) []processor.Processor {
 	processors := []processor.Processor{}
 
 	if configs := modelfile.GetConfigs(); len(configs) > 0 {
-		processors = append(processors, processor.NewModelConfigProcessor(b.store, modelspec.MediaTypeModelWeightConfig, configs))
+		mediaType := modelspec.MediaTypeModelWeightConfig
+		if cfg.Raw {
+			mediaType = modelspec.MediaTypeModelWeightConfigRaw
+		}
+		processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs))
 	}
 
 	if models := modelfile.GetModels(); len(models) > 0 {
-		processors = append(processors, processor.NewModelProcessor(b.store, modelspec.MediaTypeModelWeight, models))
+		mediaType := modelspec.MediaTypeModelWeight
+		if cfg.Raw {
+			mediaType = modelspec.MediaTypeModelWeightRaw
+		}
+		processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models))
 	}
 
 	if codes := modelfile.GetCodes(); len(codes) > 0 {
-		processors = append(processors, processor.NewCodeProcessor(b.store, modelspec.MediaTypeModelCode, codes))
+		mediaType := modelspec.MediaTypeModelCode
+		if cfg.Raw {
+			mediaType = modelspec.MediaTypeModelCodeRaw
+		}
+		processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes))
 	}
 
 	if docs := modelfile.GetDocs(); len(docs) > 0 {
-		processors = append(processors, processor.NewDocProcessor(b.store, modelspec.MediaTypeModelDoc, docs))
+		mediaType := modelspec.MediaTypeModelDoc
+		if cfg.Raw {
+			mediaType = modelspec.MediaTypeModelDocRaw
+		}
+		processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs))
 	}
 
 	return processors

pkg/backend/build/builder.go

@@ -20,11 +20,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"sync"
+	"syscall"
 	"time"
 
 	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
@@ -204,6 +206,23 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
 		applyDesc(&desc)
 	}
 
+	// Retrieve the file metadata.
+	metadata, err := getFileMetadata(path)
+	if err != nil {
+		return desc, fmt.Errorf("failed to retrieve file metadata: %w", err)
+	}
+
+	metadataStr, err := json.Marshal(metadata)
+	if err != nil {
+		return desc, fmt.Errorf("failed to marshal metadata: %w", err)
+	}
+
+	// Apply the metadata to the descriptor annotation.
+	if desc.Annotations == nil {
+		desc.Annotations = make(map[string]string)
+	}
+	desc.Annotations[modelspec.AnnotationFileMetadata] = string(metadataStr)
+
 	return desc, nil
 }
 
@@ -306,3 +325,37 @@ func splitReader(original io.Reader) (io.Reader, io.Reader) {
 
 	return r1, r2
 }
+
+// getFileMetadata retrieves metadata for a file at the given path.
+func getFileMetadata(path string) (modelspec.FileMetadata, error) {
+	var metadata modelspec.FileMetadata
+
+	info, err := os.Stat(path)
+	if err != nil {
+		return metadata, err
+	}
+
+	metadata.Name = info.Name()
+	metadata.Mode = uint32(info.Mode().Perm())
+	metadata.Size = info.Size()
+	metadata.ModTime = info.ModTime()
+	// Set Typeflag.
+	switch {
+	case info.Mode().IsRegular():
+		metadata.Typeflag = 0 // Regular file
+	case info.Mode().IsDir():
+		metadata.Typeflag = 5 // Directory
+	case info.Mode()&os.ModeSymlink != 0:
+		metadata.Typeflag = 2 // Symlink
+	default:
+		return metadata, errors.New("unknown file typeflag")
+	}
+
+	// UID and GID (Unix-specific).
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		metadata.Uid = stat.Uid
+		metadata.Gid = stat.Gid
+	}
+
+	return metadata, nil
+}

pkg/backend/build/builder_test.go

@@ -22,8 +22,10 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -126,7 +128,9 @@ func (s *BuilderTestSuite) TestBuildLayer() {
 
 		desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, hooks.NewHooks())
 		s.NoError(err)
-		s.Equal(expectedDesc, desc)
+		s.Equal(expectedDesc.MediaType, desc.MediaType)
+		s.Equal(expectedDesc.Digest, desc.Digest)
+		s.Equal(expectedDesc.Size, desc.Size)
 	})
 
 	s.Run("file not found", func() {
@@ -286,3 +290,82 @@ func TestPipeReader(t *testing.T) {
 	assert.NoError(t, err)
 	wg.Wait()
 }
+
+func createTempFile(t *testing.T, dir, pattern, content string) string {
+	t.Helper()
+	f, err := os.CreateTemp(dir, pattern)
+	assert.NoError(t, err)
+	_, err = f.WriteString(content)
+	assert.NoError(t, err)
+	err = f.Close()
+	assert.NoError(t, err)
+	return f.Name()
+}
+
+func createTempDir(t *testing.T, dir, pattern string) string {
+	t.Helper()
+	name, err := os.MkdirTemp(dir, pattern)
+	assert.NoError(t, err)
+	return name
+}
+
+func TestGetFileMetadata(t *testing.T) {
+	baseTempDir := t.TempDir()
+
+	// --- Test Case 1: Regular File ---
+	t.Run("Regular File", func(t *testing.T) {
+		content := "hello world"
+		filePath := createTempFile(t, baseTempDir, "testfile-*.txt", content)
+		fileInfo, err := os.Stat(filePath) // Get ground truth info
+		assert.NoError(t, err)
+
+		metadata, err := getFileMetadata(filePath)
+		assert.NoError(t, err)
+
+		assert.Equal(t, filepath.Base(filePath), metadata.Name)
+		assert.Equal(t, int64(len(content)), metadata.Size)
+		assert.Equal(t, uint32(fileInfo.Mode().Perm()), metadata.Mode)
+		assert.Equal(t, byte(0), metadata.Typeflag, "Typeflag should be 0 for regular file")
+		assert.WithinDuration(t, fileInfo.ModTime(), metadata.ModTime, time.Second)
+
+		// Check UID/GID only on Unix-like systems
+		if stat, ok := fileInfo.Sys().(*syscall.Stat_t); ok {
+			assert.Equal(t, stat.Uid, metadata.Uid, "UID mismatch")
+			assert.Equal(t, stat.Gid, metadata.Gid, "GID mismatch")
+		} else if runtime.GOOS != "windows" {
+			// If not windows and not syscall.Stat_t, something is unexpected
+			t.Logf("Warning: Could not get syscall.Stat_t on non-Windows OS (%s)", runtime.GOOS)
+		} else {
+			// On Windows, expect 0 as syscall.Stat_t assertion fails
+			assert.Equal(t, uint32(0), metadata.Uid, "UID should be 0 on Windows")
+			assert.Equal(t, uint32(0), metadata.Gid, "GID should be 0 on Windows")
+		}
+	})
+
+	// --- Test Case 2: Directory ---
+	t.Run("Directory", func(t *testing.T) {
+		dirPath := createTempDir(t, baseTempDir, "testdir-*")
+		dirInfo, err := os.Stat(dirPath)
+		assert.NoError(t, err)
+
+		metadata, err := getFileMetadata(dirPath)
+		assert.NoError(t, err)
+
+		assert.Equal(t, filepath.Base(dirPath), metadata.Name)
+		assert.Equal(t, dirInfo.Size(), metadata.Size)
+		assert.Equal(t, uint32(dirInfo.Mode().Perm()), metadata.Mode)
+		assert.Equal(t, byte(5), metadata.Typeflag, "Typeflag should be 5 for directory")
+		assert.WithinDuration(t, dirInfo.ModTime(), metadata.ModTime, time.Second)
+
+		// Check UID/GID only on Unix-like systems
+		if stat, ok := dirInfo.Sys().(*syscall.Stat_t); ok {
+			assert.Equal(t, stat.Uid, metadata.Uid, "UID mismatch")
+			assert.Equal(t, stat.Gid, metadata.Gid, "GID mismatch")
+		} else if runtime.GOOS != "windows" {
+			t.Logf("Warning: Could not get syscall.Stat_t on non-Windows OS (%s)", runtime.GOOS)
+		} else {
+			assert.Equal(t, uint32(0), metadata.Uid, "UID should be 0 on Windows")
+			assert.Equal(t, uint32(0), metadata.Gid, "GID should be 0 on Windows")
+		}
+	})
+}

pkg/backend/build_test.go

@@ -19,6 +19,7 @@ package backend
 import (
 	"testing"
 
+	"github.com/CloudNativeAI/modctl/pkg/config"
 	"github.com/CloudNativeAI/modctl/test/mocks/modelfile"
 
 	"github.com/stretchr/testify/assert"
@@ -32,7 +33,7 @@ func TestGetProcessors(t *testing.T) {
 	modelfile.On("GetDocs").Return([]string{"doc1", "doc2"})
 
 	b := &backend{}
-	processors := b.getProcessors(modelfile)
+	processors := b.getProcessors(modelfile, &config.Build{})
 
 	assert.Len(t, processors, 4)
 	assert.Equal(t, "config", processors[0].Name())

pkg/backend/extract.go

@@ -98,7 +98,7 @@ func extractLayer(desc ocispec.Descriptor, outputDir string, reader io.Reader) e
 		return fmt.Errorf("failed to create codec for media type %s: %w", desc.MediaType, err)
 	}
 
-	if err := codec.Decode(reader, outputDir, filepath); err != nil {
+	if err := codec.Decode(outputDir, filepath, reader, desc); err != nil {
 		return fmt.Errorf("failed to decode the layer %s to output directory: %w", desc.Digest.String(), err)
 	}
 

pkg/backend/inspect_test.go

@@ -124,7 +124,7 @@ func TestInspect(t *testing.T) {
     "format": "tensorflow",
     "paramSize": "0.5b",
     "precision": "int8",
-    "puantization": "gptq"
+    "quantization": "gptq"
   }
 }`