Skip to content

Commit 737636c

Browse files
authored
feat(attach,upload): add destination-dir flag for custom file paths (#363)
Signed-off-by: chlins <[email protected]>
1 parent 05834f5 commit 737636c

File tree

24 files changed

+155
-99
lines changed

24 files changed

+155
-99
lines changed

cmd/attach.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ func init() {
5151
flags := attachCmd.Flags()
5252
flags.StringVarP(&attachConfig.Source, "source", "s", "", "source model artifact name")
5353
flags.StringVarP(&attachConfig.Target, "target", "t", "", "target model artifact name")
54+
flags.StringVarP(&attachConfig.DestinationDir, "destination-dir", "d", "", "destination directory for the attached file should be specified as a relative path; by default, it will match the original directory of the attachment")
5455
flags.BoolVarP(&attachConfig.OutputRemote, "output-remote", "", false, "turning on this flag will output model artifact to remote registry directly")
5556
flags.BoolVarP(&attachConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
5657
flags.BoolVarP(&attachConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")

cmd/upload.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ func init() {
5353
flags.BoolVarP(&uploadConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
5454
flags.BoolVarP(&uploadConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
5555
flags.BoolVar(&uploadConfig.Raw, "raw", true, "turning on this flag will upload model artifact layer in raw format")
56+
flags.StringVar(&uploadConfig.DestinationDir, "destination-dir", "", "destination directory for the uploaded file should be specified as a relative path; by default, it will match the original directory of the uploaded file")
5657

5758
if err := viper.BindPFlags(flags); err != nil {
5859
panic(fmt.Errorf("bind cache list flags to viper: %w", err))

pkg/backend/attach.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"io"
2424
"os"
25+
pathfilepath "path/filepath"
2526
"reflect"
2627
"slices"
2728
"sort"
@@ -74,7 +75,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
7475

7576
logrus.Infof("attach: loaded source model config [%+v]", srcModelConfig)
7677

77-
proc := b.getProcessor(filepath, cfg.Raw)
78+
proc := b.getProcessor(cfg.DestinationDir, filepath, cfg.Raw)
7879
if proc == nil {
7980
return fmt.Errorf("failed to get processor for file %s", filepath)
8081
}
@@ -88,15 +89,20 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
8889
pb.Start()
8990
defer pb.Stop()
9091

92+
destPath := filepath
93+
if cfg.DestinationDir != "" {
94+
destPath = pathfilepath.Join(cfg.DestinationDir, pathfilepath.Base(filepath))
95+
}
96+
9197
layers := srcManifest.Layers
9298
// If attach a normal file, we need to process it and create a new layer.
9399
if !cfg.Config {
94100
var foundLayer *ocispec.Descriptor
95101
for _, layer := range srcManifest.Layers {
96102
if anno := layer.Annotations; anno != nil {
97-
if anno[modelspec.AnnotationFilepath] == filepath || anno[legacymodelspec.AnnotationFilepath] == filepath {
103+
if anno[modelspec.AnnotationFilepath] == destPath || anno[legacymodelspec.AnnotationFilepath] == destPath {
98104
if !cfg.Force {
99-
return fmt.Errorf("file %s already exists, please use --force to overwrite if you want to attach it forcibly", filepath)
105+
return fmt.Errorf("file %s already exists, please use --force to overwrite if you want to attach it forcibly", destPath)
100106
}
101107

102108
foundLayer = &layer
@@ -299,37 +305,37 @@ func (b *backend) getModelConfig(ctx context.Context, reference string, desc oci
299305
return &model, nil
300306
}
301307

302-
func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Processor {
308+
func (b *backend) getProcessor(destDir, filepath string, rawMediaType bool) processor.Processor {
303309
if modelfile.IsFileType(filepath, modelfile.ConfigFilePatterns) {
304310
mediaType := legacymodelspec.MediaTypeModelWeightConfig
305311
if rawMediaType {
306312
mediaType = legacymodelspec.MediaTypeModelWeightConfigRaw
307313
}
308-
return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath})
314+
return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath}, destDir)
309315
}
310316

311317
if modelfile.IsFileType(filepath, modelfile.ModelFilePatterns) {
312318
mediaType := legacymodelspec.MediaTypeModelWeight
313319
if rawMediaType {
314320
mediaType = legacymodelspec.MediaTypeModelWeightRaw
315321
}
316-
return processor.NewModelProcessor(b.store, mediaType, []string{filepath})
322+
return processor.NewModelProcessor(b.store, mediaType, []string{filepath}, destDir)
317323
}
318324

319325
if modelfile.IsFileType(filepath, modelfile.CodeFilePatterns) {
320326
mediaType := legacymodelspec.MediaTypeModelCode
321327
if rawMediaType {
322328
mediaType = legacymodelspec.MediaTypeModelCodeRaw
323329
}
324-
return processor.NewCodeProcessor(b.store, mediaType, []string{filepath})
330+
return processor.NewCodeProcessor(b.store, mediaType, []string{filepath}, destDir)
325331
}
326332

327333
if modelfile.IsFileType(filepath, modelfile.DocFilePatterns) {
328334
mediaType := legacymodelspec.MediaTypeModelDoc
329335
if rawMediaType {
330336
mediaType = legacymodelspec.MediaTypeModelDocRaw
331337
}
332-
return processor.NewDocProcessor(b.store, mediaType, []string{filepath})
338+
return processor.NewDocProcessor(b.store, mediaType, []string{filepath}, destDir)
333339
}
334340

335341
return nil

pkg/backend/attach_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func TestGetProcessor(t *testing.T) {
7373

7474
for _, tt := range tests {
7575
t.Run(tt.filepath, func(t *testing.T) {
76-
proc := b.getProcessor(tt.filepath, false)
76+
proc := b.getProcessor("", tt.filepath, false)
7777
if tt.wantType == "" {
7878
assert.Nil(t, proc)
7979
} else {

pkg/backend/build.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,31 +170,31 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build
170170
if cfg.Raw {
171171
mediaType = modelspec.MediaTypeModelWeightConfigRaw
172172
}
173-
processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs))
173+
processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs, ""))
174174
}
175175

176176
if models := modelfile.GetModels(); len(models) > 0 {
177177
mediaType := modelspec.MediaTypeModelWeight
178178
if cfg.Raw {
179179
mediaType = modelspec.MediaTypeModelWeightRaw
180180
}
181-
processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models))
181+
processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models, ""))
182182
}
183183

184184
if codes := modelfile.GetCodes(); len(codes) > 0 {
185185
mediaType := modelspec.MediaTypeModelCode
186186
if cfg.Raw {
187187
mediaType = modelspec.MediaTypeModelCodeRaw
188188
}
189-
processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes))
189+
processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes, ""))
190190
}
191191

192192
if docs := modelfile.GetDocs(); len(docs) > 0 {
193193
mediaType := modelspec.MediaTypeModelDoc
194194
if cfg.Raw {
195195
mediaType = modelspec.MediaTypeModelDocRaw
196196
}
197-
processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs))
197+
processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs, ""))
198198
}
199199

200200
return processors

pkg/backend/build/builder.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ const (
5858
// Builder is an interface for building artifacts.
5959
type Builder interface {
6060
// BuildLayer builds the layer blob from the given file path.
61-
BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error)
61+
BuildLayer(ctx context.Context, mediaType, workDir, path, destPath string, hooks hooks.Hooks) (ocispec.Descriptor, error)
6262

6363
// BuildConfig builds the config blob of the artifact.
6464
BuildConfig(ctx context.Context, config modelspec.Model, hooks hooks.Hooks) (ocispec.Descriptor, error)
@@ -69,7 +69,7 @@ type Builder interface {
6969

7070
type OutputStrategy interface {
7171
// OutputLayer outputs the layer blob to the storage (local or remote).
72-
OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error)
72+
OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error)
7373

7474
// OutputConfig outputs the config blob to the storage (local or remote).
7575
OutputConfig(ctx context.Context, mediaType, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error)
@@ -122,7 +122,7 @@ type abstractBuilder struct {
122122
interceptor interceptor.Interceptor
123123
}
124124

125-
func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error) {
125+
func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path, destPath string, hooks hooks.Hooks) (ocispec.Descriptor, error) {
126126
info, err := os.Stat(path)
127127
if err != nil {
128128
return ocispec.Descriptor{}, fmt.Errorf("failed to get file info: %w", err)
@@ -179,7 +179,7 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
179179
}()
180180
}
181181

182-
desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, digest, size, reader, hooks)
182+
desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, destPath, digest, size, reader, hooks)
183183
if err != nil {
184184
return desc, err
185185
}

pkg/backend/build/builder_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,23 +123,23 @@ func (s *BuilderTestSuite) TestBuildLayer() {
123123
Size: 100,
124124
}
125125

126-
s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything).
126+
s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", "", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything).
127127
Return(expectedDesc, nil)
128128

129-
desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, hooks.NewHooks())
129+
desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, "", hooks.NewHooks())
130130
s.NoError(err)
131131
s.Equal(expectedDesc.MediaType, desc.MediaType)
132132
s.Equal(expectedDesc.Digest, desc.Digest)
133133
s.Equal(expectedDesc.Size, desc.Size)
134134
})
135135

136136
s.Run("file not found", func() {
137-
_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), hooks.NewHooks())
137+
_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), "", hooks.NewHooks())
138138
s.Error(err)
139139
})
140140

141141
s.Run("directory not supported", func() {
142-
_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, hooks.NewHooks())
142+
_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, "", hooks.NewHooks())
143143
s.Error(err)
144144
s.True(strings.Contains(err.Error(), "is a directory and not supported yet"))
145145
})

pkg/backend/build/local.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,20 +46,24 @@ type localOutput struct {
4646
}
4747

4848
// OutputLayer outputs the layer blob to the local storage.
49-
func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
49+
func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
5050
reader = hooks.OnStart(relPath, size, reader)
5151
digest, size, err := lo.store.PushBlob(ctx, lo.repo, reader, ocispec.Descriptor{})
5252
if err != nil {
5353
hooks.OnError(relPath, err)
5454
return ocispec.Descriptor{}, fmt.Errorf("failed to push blob to storage: %w", err)
5555
}
5656

57+
if destPath == "" {
58+
destPath = relPath
59+
}
60+
5761
desc := ocispec.Descriptor{
5862
MediaType: mediaType,
5963
Digest: godigest.Digest(digest),
6064
Size: size,
6165
Annotations: map[string]string{
62-
modelspec.AnnotationFilepath: relPath,
66+
modelspec.AnnotationFilepath: destPath,
6367
},
6468
}
6569

pkg/backend/build/local_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ func (s *LocalOutputTestSuite) TestOutputLayer() {
7272
s.mockStorage.On("PushBlob", s.ctx, "test-repo", mock.Anything, ocispec.Descriptor{}).
7373
Return(expectedDigest, expectedSize, nil).Once()
7474

75-
desc, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "test-file.txt", expectedDigest, expectedSize, reader, hooks.NewHooks())
75+
desc, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "test-file.txt", "", expectedDigest, expectedSize, reader, hooks.NewHooks())
7676

7777
s.NoError(err)
7878
s.Equal("test/mediatype", desc.MediaType)
@@ -88,7 +88,7 @@ func (s *LocalOutputTestSuite) TestOutputLayer() {
8888
s.mockStorage.On("PushBlob", s.ctx, "test-repo", mock.Anything, ocispec.Descriptor{}).
8989
Return("", int64(0), errors.New("storage error")).Once()
9090

91-
_, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "/work", "test-file.txt", int64(0), reader, hooks.NewHooks())
91+
_, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "/work", "test-file.txt", "", int64(0), reader, hooks.NewHooks())
9292

9393
s.Error(err)
9494
s.Contains(err.Error(), "failed to push blob to storage")

pkg/backend/build/remote.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,17 @@ type remoteOutput struct {
5151
}
5252

5353
// OutputLayer outputs the layer blob to the remote storage.
54-
func (ro *remoteOutput) OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
54+
func (ro *remoteOutput) OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
55+
if destPath == "" {
56+
destPath = relPath
57+
}
58+
5559
desc := ocispec.Descriptor{
5660
MediaType: mediaType,
5761
Digest: godigest.Digest(digest),
5862
Size: size,
5963
Annotations: map[string]string{
60-
modelspec.AnnotationFilepath: relPath,
64+
modelspec.AnnotationFilepath: destPath,
6165
},
6266
}
6367

0 commit comments

Comments
 (0)