Skip to content

Commit f54fd22

Browse files
chlinschlins
committed
feat: add the upload command for uploading blob in advance
Signed-off-by: chlins <[email protected]>
1 parent b908218 commit f54fd22

File tree

7 files changed

+183
-7
lines changed

7 files changed

+183
-7
lines changed

‎cmd/root.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,5 +142,6 @@ func init() {
142142
rootCmd.AddCommand(tagCmd)
143143
rootCmd.AddCommand(fetchCmd)
144144
rootCmd.AddCommand(attachCmd)
145+
rootCmd.AddCommand(uploadCmd)
145146
rootCmd.AddCommand(modelfile.RootCmd)
146147
}

‎cmd/upload.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright 2025 The CNAI Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
"github.com/spf13/cobra"
24+
"github.com/spf13/viper"
25+
26+
"github.com/CloudNativeAI/modctl/pkg/backend"
27+
"github.com/CloudNativeAI/modctl/pkg/config"
28+
)
29+
30+
var uploadConfig = config.NewUpload()
31+
32+
// uploadCmd represents the modctl command for upload.
33+
var uploadCmd = &cobra.Command{
34+
Use: "upload [flags] <file>",
35+
Short: "A command line tool for modctl upload",
36+
Args: cobra.ExactArgs(1),
37+
DisableAutoGenTag: true,
38+
SilenceUsage: true,
39+
FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true},
40+
RunE: func(cmd *cobra.Command, args []string) error {
41+
if err := uploadConfig.Validate(); err != nil {
42+
return err
43+
}
44+
45+
return runUpload(context.Background(), args[0])
46+
},
47+
}
48+
49+
// init initializes upload command.
50+
func init() {
51+
flags := uploadCmd.Flags()
52+
flags.StringVarP(&uploadConfig.Repo, "repo", "", "", "target model artifact repository name")
53+
flags.BoolVarP(&uploadConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
54+
flags.BoolVarP(&uploadConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
55+
flags.BoolVar(&uploadConfig.Raw, "raw", false, "turning on this flag will upload model artifact layer in raw format")
56+
57+
if err := viper.BindPFlags(flags); err != nil {
58+
panic(fmt.Errorf("bind cache list flags to viper: %w", err))
59+
}
60+
}
61+
62+
// runUpload runs the upload modctl.
63+
func runUpload(ctx context.Context, filepath string) error {
64+
b, err := backend.New(rootConfig.StoargeDir)
65+
if err != nil {
66+
return err
67+
}
68+
69+
if err := b.Upload(ctx, filepath, uploadConfig); err != nil {
70+
return err
71+
}
72+
73+
fmt.Printf("Successfully uploaded %s to model artifact repository: %s\n", filepath, uploadConfig.Repo)
74+
return nil
75+
}

‎pkg/backend/attach.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
100100
}
101101
}
102102

103-
proc := b.getProcessor(filepath, cfg)
103+
proc := b.getProcessor(filepath, cfg.Raw)
104104
if proc == nil {
105105
return fmt.Errorf("failed to get processor for file %s", filepath)
106106
}
@@ -272,34 +272,34 @@ func (b *backend) getModelConfig(ctx context.Context, reference string, desc oci
272272
return &model, nil
273273
}
274274

275-
func (b *backend) getProcessor(filepath string, cfg *config.Attach) processor.Processor {
275+
func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Processor {
276276
if modelfile.IsFileType(filepath, modelfile.ConfigFilePatterns) {
277277
mediaType := modelspec.MediaTypeModelWeightConfig
278-
if cfg.Raw {
278+
if rawMediaType {
279279
mediaType = modelspec.MediaTypeModelWeightConfigRaw
280280
}
281281
return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath})
282282
}
283283

284284
if modelfile.IsFileType(filepath, modelfile.ModelFilePatterns) {
285285
mediaType := modelspec.MediaTypeModelWeight
286-
if cfg.Raw {
286+
if rawMediaType {
287287
mediaType = modelspec.MediaTypeModelWeightRaw
288288
}
289289
return processor.NewModelProcessor(b.store, mediaType, []string{filepath})
290290
}
291291

292292
if modelfile.IsFileType(filepath, modelfile.CodeFilePatterns) {
293293
mediaType := modelspec.MediaTypeModelCode
294-
if cfg.Raw {
294+
if rawMediaType {
295295
mediaType = modelspec.MediaTypeModelCodeRaw
296296
}
297297
return processor.NewCodeProcessor(b.store, mediaType, []string{filepath})
298298
}
299299

300300
if modelfile.IsFileType(filepath, modelfile.DocFilePatterns) {
301301
mediaType := modelspec.MediaTypeModelDoc
302-
if cfg.Raw {
302+
if rawMediaType {
303303
mediaType = modelspec.MediaTypeModelDocRaw
304304
}
305305
return processor.NewDocProcessor(b.store, mediaType, []string{filepath})

‎pkg/backend/attach_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func TestGetProcessor(t *testing.T) {
7373

7474
for _, tt := range tests {
7575
t.Run(tt.filepath, func(t *testing.T) {
76-
proc := b.getProcessor(tt.filepath, &config.Attach{})
76+
proc := b.getProcessor(tt.filepath, false)
7777
if tt.wantType == "" {
7878
assert.Nil(t, proc)
7979
} else {

‎pkg/backend/backend.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ type Backend interface {
3434
// Attach attaches user materials into the model artifact which follows the Model Spec.
3535
Attach(ctx context.Context, filepath string, cfg *config.Attach) error
3636

37+
// Upload uploads the file to a model artifact repository in advance, but will not push config and manifest.
38+
Upload(ctx context.Context, filepath string, cfg *config.Upload) error
39+
3740
// Build builds the user materials into the model artifact which follows the Model Spec.
3841
Build(ctx context.Context, modelfilePath, workDir, target string, cfg *config.Build) error
3942

‎pkg/backend/upload.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright 2025 The CNAI Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package backend
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
internalpb "github.com/CloudNativeAI/modctl/internal/pb"
24+
"github.com/CloudNativeAI/modctl/pkg/backend/build"
25+
"github.com/CloudNativeAI/modctl/pkg/backend/processor"
26+
"github.com/CloudNativeAI/modctl/pkg/config"
27+
)
28+
29+
// Upload uploads the file to a model artifact repository in advance, but will not push config and manifest.
30+
func (b *backend) Upload(ctx context.Context, filepath string, cfg *config.Upload) error {
31+
proc := b.getProcessor(filepath, cfg.Raw)
32+
if proc == nil {
33+
return fmt.Errorf("failed to get processor for file %s", filepath)
34+
}
35+
36+
opts := []build.Option{
37+
build.WithPlainHTTP(cfg.PlainHTTP),
38+
build.WithInsecure(cfg.Insecure),
39+
}
40+
builder, err := build.NewBuilder(build.OutputTypeRemote, b.store, cfg.Repo, "", opts...)
41+
if err != nil {
42+
return fmt.Errorf("failed to create builder: %w", err)
43+
}
44+
45+
pb := internalpb.NewProgressBar()
46+
pb.Start()
47+
defer pb.Stop()
48+
49+
if _, err = proc.Process(ctx, builder, ".", processor.WithProgressTracker(pb)); err != nil {
50+
return fmt.Errorf("failed to process layers: %w", err)
51+
}
52+
53+
return nil
54+
}

‎pkg/config/upload.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2025 The CNAI Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package config
18+
19+
import "errors"
20+
21+
type Upload struct {
22+
Repo string
23+
PlainHTTP bool
24+
Insecure bool
25+
Raw bool
26+
}
27+
28+
func NewUpload() *Upload {
29+
return &Upload{
30+
Repo: "",
31+
PlainHTTP: false,
32+
Insecure: false,
33+
Raw: false,
34+
}
35+
}
36+
37+
func (u *Upload) Validate() error {
38+
if u.Repo == "" {
39+
return errors.New("repo is required")
40+
}
41+
42+
return nil
43+
}

0 commit comments

Comments
 (0)