feat: support the upload command and optimize the log (#215)

Signed-off-by: chlins <[email protected]>

Author: chlins

cmd/modelfile/modelfile.go

@@ -17,8 +17,6 @@
 package modelfile
 
 import (
-	"github.com/sirupsen/logrus"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -32,8 +30,6 @@ var RootCmd = &cobra.Command{
 	SilenceUsage:       true,
 	FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		logrus.Debug("modctl modelfile is running")
-
 		return nil
 	},
 }

cmd/root.go

@@ -142,5 +142,6 @@ func init() {
 	rootCmd.AddCommand(tagCmd)
 	rootCmd.AddCommand(fetchCmd)
 	rootCmd.AddCommand(attachCmd)
+	rootCmd.AddCommand(uploadCmd)
 	rootCmd.AddCommand(modelfile.RootCmd)
 }

cmd/upload.go

@@ -0,0 +1,75 @@
+/*
+ *     Copyright 2025 The CNAI Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/CloudNativeAI/modctl/pkg/backend"
+	"github.com/CloudNativeAI/modctl/pkg/config"
+)
+
+var uploadConfig = config.NewUpload()
+
+// uploadCmd represents the modctl command for upload.
+var uploadCmd = &cobra.Command{
+	Use:                "upload [flags] <file>",
+	Short:              "A command line tool for modctl upload",
+	Args:               cobra.ExactArgs(1),
+	DisableAutoGenTag:  true,
+	SilenceUsage:       true,
+	FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if err := uploadConfig.Validate(); err != nil {
+			return err
+		}
+
+		return runUpload(context.Background(), args[0])
+	},
+}
+
+// init initializes upload command.
+func init() {
+	flags := uploadCmd.Flags()
+	flags.StringVarP(&uploadConfig.Repo, "repo", "", "", "target model artifact repository name")
+	flags.BoolVarP(&uploadConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
+	flags.BoolVarP(&uploadConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
+	flags.BoolVar(&uploadConfig.Raw, "raw", false, "turning on this flag will upload model artifact layer in raw format")
+
+	if err := viper.BindPFlags(flags); err != nil {
+		panic(fmt.Errorf("bind cache list flags to viper: %w", err))
+	}
+}
+
+// runUpload runs the upload modctl.
+func runUpload(ctx context.Context, filepath string) error {
+	b, err := backend.New(rootConfig.StoargeDir)
+	if err != nil {
+		return err
+	}
+
+	if err := b.Upload(ctx, filepath, uploadConfig); err != nil {
+		return err
+	}
+
+	fmt.Printf("Successfully uploaded %s to model artifact repository: %s\n", filepath, uploadConfig.Repo)
+	return nil
+}

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/vbauerster/mpb/v8 v8.10.2
 	golang.org/x/crypto v0.39.0
 	golang.org/x/sync v0.15.0
+	golang.org/x/sys v0.33.0
 	google.golang.org/grpc v1.73.0
 	oras.land/oras-go/v2 v2.6.0
 )
@@ -111,7 +112,6 @@ require (
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.41.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect

pkg/backend/attach.go

@@ -60,7 +60,7 @@ var (
 
 // Attach attaches user materials into the model artifact which follows the Model Spec.
 func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attach) error {
-	logrus.Infof("attaching file %s, cfg: %+v", filepath, cfg)
+	logrus.Infof("attach: starting attach operation for file %s [config: %+v]", filepath, cfg)
 	srcManifest, err := b.getManifest(ctx, cfg.Source, cfg.OutputRemote, cfg.PlainHTTP, cfg.Insecure)
 	if err != nil {
 		return fmt.Errorf("failed to get source manifest: %w", err)
@@ -71,7 +71,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		return fmt.Errorf("failed to get source model config: %w", err)
 	}
 
-	logrus.Infof("source model config: %+v", srcModelConfig)
+	logrus.Infof("attach: loaded source model config [%+v]", srcModelConfig)
 
 	var foundLayer *ocispec.Descriptor
 	for _, layer := range srcManifest.Layers {
@@ -87,7 +87,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		}
 	}
 
-	logrus.Infof("found original layer: %+v", foundLayer)
+	logrus.Infof("attach: found existing layer for file %s [%+v]", filepath, foundLayer)
 
 	layers := srcManifest.Layers
 	if foundLayer != nil {
@@ -100,7 +100,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		}
 	}
 
-	proc := b.getProcessor(filepath, cfg)
+	proc := b.getProcessor(filepath, cfg.Raw)
 	if proc == nil {
 		return fmt.Errorf("failed to get processor for file %s", filepath)
 	}
@@ -123,7 +123,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 	layers = append(layers, newLayers...)
 	sortLayers(layers)
 
-	logrus.Infof("new sorted layers: %+v", layers)
+	logrus.Debugf("attach: generated sorted layers [layers: %+v]", layers)
 
 	diffIDs := []godigest.Digest{}
 	for _, layer := range layers {
@@ -145,7 +145,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		Name:         srcModelConfig.Descriptor.Name,
 	}
 
-	logrus.Infof("new model config: %+v", modelConfig)
+	logrus.Infof("attach: built model config [%+v]", modelConfig)
 
 	configDesc, err := builder.BuildConfig(ctx, layers, modelConfig, hooks.NewHooks(
 		hooks.WithOnStart(func(name string, size int64, reader io.Reader) io.Reader {
@@ -178,6 +178,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 		return fmt.Errorf("failed to build model manifest: %w", err)
 	}
 
+	logrus.Infof("attach: successfully attached file %s", filepath)
 	return nil
 }
 
@@ -272,34 +273,34 @@ func (b *backend) getModelConfig(ctx context.Context, reference string, desc oci
 	return &model, nil
 }
 
-func (b *backend) getProcessor(filepath string, cfg *config.Attach) processor.Processor {
+func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Processor {
 	if modelfile.IsFileType(filepath, modelfile.ConfigFilePatterns) {
 		mediaType := modelspec.MediaTypeModelWeightConfig
-		if cfg.Raw {
+		if rawMediaType {
 			mediaType = modelspec.MediaTypeModelWeightConfigRaw
 		}
 		return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath})
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.ModelFilePatterns) {
 		mediaType := modelspec.MediaTypeModelWeight
-		if cfg.Raw {
+		if rawMediaType {
 			mediaType = modelspec.MediaTypeModelWeightRaw
 		}
 		return processor.NewModelProcessor(b.store, mediaType, []string{filepath})
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.CodeFilePatterns) {
 		mediaType := modelspec.MediaTypeModelCode
-		if cfg.Raw {
+		if rawMediaType {
 			mediaType = modelspec.MediaTypeModelCodeRaw
 		}
 		return processor.NewCodeProcessor(b.store, mediaType, []string{filepath})
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.DocFilePatterns) {
 		mediaType := modelspec.MediaTypeModelDoc
-		if cfg.Raw {
+		if rawMediaType {
 			mediaType = modelspec.MediaTypeModelDocRaw
 		}
 		return processor.NewDocProcessor(b.store, mediaType, []string{filepath})

pkg/backend/attach_test.go

@@ -73,7 +73,7 @@ func TestGetProcessor(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.filepath, func(t *testing.T) {
-			proc := b.getProcessor(tt.filepath, &config.Attach{})
+			proc := b.getProcessor(tt.filepath, false)
 			if tt.wantType == "" {
 				assert.Nil(t, proc)
 			} else {

pkg/backend/backend.go

@@ -34,6 +34,9 @@ type Backend interface {
 	// Attach attaches user materials into the model artifact which follows the Model Spec.
 	Attach(ctx context.Context, filepath string, cfg *config.Attach) error
 
+	// Upload uploads the file to a model artifact repository in advance, but will not push config and manifest.
+	Upload(ctx context.Context, filepath string, cfg *config.Upload) error
+
 	// Build builds the user materials into the model artifact which follows the Model Spec.
 	Build(ctx context.Context, modelfilePath, workDir, target string, cfg *config.Build) error
 

pkg/backend/build.go

@@ -46,7 +46,7 @@ const (
 
 // Build builds the user materials into the model artifact which follows the Model Spec.
 func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target string, cfg *config.Build) error {
-	logrus.Infof("building model artifact: %s, cfg: %+v", target, cfg)
+	logrus.Infof("build: starting build operation for target %s [config: %+v]", target, cfg)
 	// parse the repo name and tag name from target.
 	ref, err := ParseReference(target)
 	if err != nil {
@@ -99,7 +99,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 
 	layers = append(layers, layerDescs...)
 
-	logrus.Infof("model artifact layers: %+v", layers)
+	logrus.Infof("build: processed layers for artifact [count: %d, layers: %+v]", len(layers), layers)
 
 	revision := sourceInfo.Commit
 	if revision != "" && sourceInfo.Dirty {
@@ -118,7 +118,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 		SourceRevision: revision,
 	}
 
-	logrus.Infof("model artifact config: %+v", modelConfig)
+	logrus.Infof("build: built model config [family: %s, name: %s, format: %s]", modelConfig.Family, modelConfig.Name, modelConfig.Format)
 
 	var configDesc ocispec.Descriptor
 	// Build the model config.
@@ -157,6 +157,7 @@ func (b *backend) Build(ctx context.Context, modelfilePath, workDir, target stri
 		return fmt.Errorf("failed to build model manifest: %w", err)
 	}
 
+	logrus.Infof("build: successfully built model artifact %s", target)
 	return nil
 }