Skip to content

Commit 789f587

Browse files
darthShadowdarthShadow
committed
vfs: set appropriate file & directory metadata from mount
vfs,union,local: set metadata for automatically created/cloned directories Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>
1 parent 998a4bc commit 789f587

File tree

7 files changed

+172
-64
lines changed

7 files changed

+172
-64
lines changed

‎backend/local/clone_darwin.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
4343

4444
// Create destination
4545
dstObj := f.newObject(remote)
46-
err = dstObj.mkdirAll()
46+
err = dstObj.mkdirAll(ctx)
4747
if err != nil {
4848
return nil, err
4949
}

‎backend/local/local.go‎

Lines changed: 81 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -756,23 +756,82 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
756756
return f.Put(ctx, in, src, options...)
757757
}
758758

759-
// Mkdir creates the directory if it doesn't exist
760-
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
761-
localPath := f.localPath(dir)
762-
err := file.MkdirAll(localPath, 0777)
759+
// mkParentDir creates the parent of the remote path if it doesn't exist
760+
func (f *Fs) mkParentDir(ctx context.Context, remote string, metadata fs.Metadata) error {
761+
remoteDir := filepath.Dir(remote)
762+
// Avoid infinite recursion
763+
if remoteDir == remote {
764+
return nil
765+
}
766+
767+
localPath := f.localPath(remote)
768+
localDirPath := filepath.Dir(localPath)
769+
_, err := f.lstat(localDirPath)
770+
if err == nil {
771+
return nil
772+
}
773+
if !errors.Is(err, os.ErrNotExist) {
774+
return fmt.Errorf("mkParentDir: %s: failed to read info: %s: %w", remote, localDirPath, err)
775+
}
776+
777+
err = f.mkParentDir(ctx, remoteDir, metadata)
763778
if err != nil {
764-
return err
779+
return fmt.Errorf("mkParentDir: %s: failed to make parent directory: %s: %w", remote, remoteDir, err)
765780
}
766-
if dir == "" {
767-
fi, err := f.lstat(localPath)
781+
782+
err = file.MkdirAll(localDirPath, 0777)
783+
if err != nil {
784+
return fmt.Errorf("mkParentDir: %s: failed to make directory: %s: %w", remote, localPath, err)
785+
}
786+
787+
if metadata != nil && remoteDir != "" && remoteDir != "." && remoteDir != ".." && remoteDir != "/" {
788+
fi, err := f.lstat(localDirPath)
768789
if err != nil {
769-
return err
790+
return fmt.Errorf("mkParentDir: %s: failed to read info after creating directory: %s: %w", remote, localDirPath, err)
791+
}
792+
d := f.newDirectory(remoteDir, fi)
793+
err = d.writeMetadata(metadata)
794+
if err != nil {
795+
return fmt.Errorf("mkParentDir: %s: failed to set metadata: %s: %w", remote, localPath, err)
770796
}
771-
f.dev = readDevice(fi, f.opt.OneFileSystem)
772797
}
798+
773799
return nil
774800
}
775801

802+
func (f *Fs) mkdir(ctx context.Context, dir string, metadata fs.Metadata) (os.FileInfo, error) {
803+
localPath := f.localPath(dir)
804+
fi, err := f.lstat(localPath)
805+
if errors.Is(err, os.ErrNotExist) {
806+
err := f.mkParentDir(ctx, dir, metadata)
807+
if err != nil {
808+
return nil, fmt.Errorf("mkdir: %s: failed to make parent directory: %w", dir, err)
809+
}
810+
err = file.MkdirAll(localPath, 0777)
811+
if err != nil {
812+
return nil, err
813+
}
814+
fi, err = f.lstat(localPath)
815+
if err != nil {
816+
return nil, fmt.Errorf("mkdir: %s: failed to re-read info: %s: %w", dir, localPath, err)
817+
}
818+
} else if err != nil {
819+
return nil, fmt.Errorf("mkdir: %s: failed to read info: %s: %w", dir, localPath, err)
820+
}
821+
822+
if dir == "" {
823+
f.dev = readDevice(fi, f.opt.OneFileSystem)
824+
}
825+
826+
return fi, nil
827+
}
828+
829+
// Mkdir creates the directory if it doesn't exist
830+
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
831+
_, err := f.mkdir(ctx, dir, nil)
832+
return err
833+
}
834+
776835
// DirSetModTime sets the directory modtime for dir
777836
func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
778837
o := Object{
@@ -792,19 +851,9 @@ func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) e
792851
// It returns the directory that was created.
793852
func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
794853
// Find and or create the directory
795-
localPath := f.localPath(dir)
796-
fi, err := f.lstat(localPath)
797-
if errors.Is(err, os.ErrNotExist) {
798-
err := f.Mkdir(ctx, dir)
799-
if err != nil {
800-
return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
801-
}
802-
fi, err = f.lstat(localPath)
803-
if err != nil {
804-
return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
805-
}
806-
} else if err != nil {
807-
return nil, err
854+
fi, err := f.mkdir(ctx, dir, metadata)
855+
if err != nil {
856+
return nil, fmt.Errorf("mkdir metadata: %w", err)
808857
}
809858

810859
// Create directory object
@@ -814,14 +863,15 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
814863
if metadata != nil {
815864
err = d.writeMetadata(metadata)
816865
if err != nil {
817-
return nil, fmt.Errorf("failed to set metadata on directory: %w", err)
866+
return nil, fmt.Errorf("mkdir metadata: failed to set metadata on directory: %w", err)
818867
}
819868
// Re-read info now we have finished setting stuff
820869
err = d.lstat()
821870
if err != nil {
822871
return nil, fmt.Errorf("mkdir metadata: failed to re-read info: %w", err)
823872
}
824873
}
874+
825875
return d, nil
826876
}
827877

@@ -936,7 +986,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
936986
}
937987

938988
// Create destination
939-
err = dstObj.mkdirAll()
989+
err = dstObj.mkdirAll(ctx)
940990
if err != nil {
941991
return nil, err
942992
}
@@ -1001,8 +1051,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
10011051
}
10021052

10031053
// Create parent of destination
1004-
dstParentPath := filepath.Dir(dstPath)
1005-
err = file.MkdirAll(dstParentPath, 0777)
1054+
metadata := fs.GetConfig(ctx).MetadataSet
1055+
err = f.mkParentDir(ctx, dstRemote, metadata)
10061056
if err != nil {
10071057
return err
10081058
}
@@ -1345,9 +1395,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
13451395
}
13461396

13471397
// mkdirAll makes all the directories needed to store the object
1348-
func (o *Object) mkdirAll() error {
1349-
dir := filepath.Dir(o.path)
1350-
return file.MkdirAll(dir, 0777)
1398+
func (o *Object) mkdirAll(ctx context.Context) error {
1399+
metadata := fs.GetConfig(ctx).MetadataSet
1400+
return o.fs.mkParentDir(ctx, o.remote, metadata)
13511401
}
13521402

13531403
type nopWriterCloser struct {
@@ -1376,7 +1426,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
13761426
}
13771427
}
13781428

1379-
err = o.mkdirAll()
1429+
err = o.mkdirAll(ctx)
13801430
if err != nil {
13811431
return err
13821432
}
@@ -1495,7 +1545,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
14951545
// Temporary Object under construction
14961546
o := f.newObject(remote)
14971547

1498-
err := o.mkdirAll()
1548+
err := o.mkdirAll(ctx)
14991549
if err != nil {
15001550
return nil, err
15011551
}

‎backend/union/entry.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func (d *Directory) candidates() []upstream.Entry {
6868
// return an error or update the object properly (rather than e.g. calling panic).
6969
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
7070
entries, err := o.fs.actionEntries(o.candidates()...)
71-
if err == fs.ErrorPermissionDenied {
71+
if errors.Is(err, fs.ErrorPermissionDenied) {
7272
// There are no candidates in this object which can be written to
7373
// So attempt to create a new object instead
7474
newO, err := o.fs.put(ctx, in, src, false, options...)

‎backend/union/union.go‎

Lines changed: 47 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
129129
if err != nil {
130130
// If none of the backends can have empty directories then
131131
// don't complain about directories not being found
132-
if !f.features.CanHaveEmptyDirectories && err == fs.ErrorObjectNotFound {
132+
if !f.features.CanHaveEmptyDirectories && errors.Is(err, fs.ErrorObjectNotFound) {
133133
return nil
134134
}
135135
return err
@@ -150,12 +150,12 @@ func (f *Fs) Hashes() hash.Set {
150150
}
151151

152152
// mkdir makes the directory passed in and returns the upstreams used
153-
func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
153+
func (f *Fs) mkdir(ctx context.Context, dir string, withMetadata bool) ([]*upstream.Fs, error) {
154154
upstreams, err := f.create(ctx, dir)
155-
if err == fs.ErrorObjectNotFound {
155+
if errors.Is(err, fs.ErrorObjectNotFound) {
156156
parent := parentDir(dir)
157157
if dir != parent {
158-
upstreams, err = f.mkdir(ctx, parent)
158+
upstreams, err = f.mkdir(ctx, parent, withMetadata)
159159
} else if dir == "" {
160160
// If root dirs not created then create them
161161
upstreams, err = f.upstreams, nil
@@ -164,17 +164,38 @@ func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
164164
if err != nil {
165165
return nil, err
166166
}
167+
168+
metadataSet := fs.GetConfig(ctx).MetadataSet
169+
167170
errs := Errors(make([]error, len(upstreams)))
168171
multithread(len(upstreams), func(i int) {
169-
err := upstreams[i].Mkdir(ctx, dir)
170-
if err != nil {
171-
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
172+
u := upstreams[i]
173+
if withMetadata && metadataSet != nil {
174+
if do := u.Features().MkdirMetadata; do != nil {
175+
_, err := do(ctx, dir, metadataSet)
176+
if err != nil {
177+
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
178+
}
179+
180+
} else {
181+
// Just do Mkdir on upstreams which don't support MkdirMetadata
182+
err := u.Mkdir(ctx, dir)
183+
if err != nil {
184+
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
185+
}
186+
}
187+
} else {
188+
err := u.Mkdir(ctx, dir)
189+
if err != nil {
190+
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
191+
}
172192
}
173193
})
174194
err = errs.Err()
175195
if err != nil {
176196
return nil, err
177197
}
198+
178199
// If created roots then choose one
179200
if dir == "" {
180201
upstreams, err = f.create(ctx, dir)
@@ -184,7 +205,7 @@ func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
184205

185206
// Mkdir makes the root directory of the Fs object
186207
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
187-
_, err := f.mkdir(ctx, dir)
208+
_, err := f.mkdir(ctx, dir, false)
188209
return err
189210
}
190211

@@ -201,19 +222,19 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
201222
if do := u.Features().MkdirMetadata; do != nil {
202223
newDir, err := do(ctx, dir, metadata)
203224
if err != nil {
204-
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
225+
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
205226
} else {
206227
entries[i], err = u.WrapEntry(newDir)
207228
if err != nil {
208-
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
229+
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
209230
}
210231
}
211232

212233
} else {
213234
// Just do Mkdir on upstreams which don't support MkdirMetadata
214235
err := u.Mkdir(ctx, dir)
215236
if err != nil {
216-
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
237+
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
217238
}
218239
}
219240
})
@@ -533,8 +554,8 @@ func multiReader(n int, in io.Reader) ([]io.Reader, <-chan error) {
533554
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
534555
srcPath := src.Remote()
535556
upstreams, err := f.create(ctx, srcPath)
536-
if err == fs.ErrorObjectNotFound {
537-
upstreams, err = f.mkdir(ctx, parentDir(srcPath))
557+
if errors.Is(err, fs.ErrorObjectNotFound) {
558+
upstreams, err = f.mkdir(ctx, parentDir(srcPath), true)
538559
}
539560
if err != nil {
540561
return nil, err
@@ -593,10 +614,10 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
593614
// nil and the error
594615
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
595616
o, err := f.NewObject(ctx, src.Remote())
596-
switch err {
597-
case nil:
617+
switch {
618+
case err == nil:
598619
return o, o.Update(ctx, in, src, options...)
599-
case fs.ErrorObjectNotFound:
620+
case errors.Is(err, fs.ErrorObjectNotFound):
600621
return f.put(ctx, in, src, false, options...)
601622
default:
602623
return nil, err
@@ -610,10 +631,10 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
610631
// nil and the error
611632
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
612633
o, err := f.NewObject(ctx, src.Remote())
613-
switch err {
614-
case nil:
634+
switch {
635+
case err == nil:
615636
return o, o.Update(ctx, in, src, options...)
616-
case fs.ErrorObjectNotFound:
637+
case errors.Is(err, fs.ErrorObjectNotFound):
617638
return f.put(ctx, in, src, true, options...)
618639
default:
619640
return nil, err
@@ -782,7 +803,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
782803
multithread(len(f.upstreams), func(i int) {
783804
u := f.upstreams[i]
784805
o, err := u.NewObject(ctx, remote)
785-
if err != nil && err != fs.ErrorObjectNotFound {
806+
if err != nil && !errors.Is(err, fs.ErrorObjectNotFound) {
786807
errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
787808
return
788809
}
@@ -832,7 +853,7 @@ func (f *Fs) searchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
832853
}
833854

834855
func (f *Fs) mergeDirEntries(entriesList [][]upstream.Entry) (fs.DirEntries, error) {
835-
entryMap := make(map[string]([]upstream.Entry))
856+
entryMap := make(map[string][]upstream.Entry)
836857
for _, en := range entriesList {
837858
if en == nil {
838859
continue
@@ -846,8 +867,8 @@ func (f *Fs) mergeDirEntries(entriesList [][]upstream.Entry) (fs.DirEntries, err
846867
}
847868
}
848869
var entries fs.DirEntries
849-
for path := range entryMap {
850-
e, err := f.wrapEntries(entryMap[path]...)
870+
for entryPath := range entryMap {
871+
e, err := f.wrapEntries(entryMap[entryPath]...)
851872
if err != nil {
852873
return nil, err
853874
}
@@ -929,11 +950,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
929950
var usedUpstreams []*upstream.Fs
930951
var fserr error
931952
for i, err := range errs {
932-
if err != nil && err != fs.ErrorIsFile {
953+
if err != nil && !errors.Is(err, fs.ErrorIsFile) {
933954
return nil, err
934955
}
935956
// Only the upstreams returns ErrorIsFile would be used if any
936-
if err == fs.ErrorIsFile {
957+
if errors.Is(err, fs.ErrorIsFile) {
937958
usedUpstreams = append(usedUpstreams, upstreams[i])
938959
fserr = fs.ErrorIsFile
939960
}
@@ -949,7 +970,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
949970
upstreams: usedUpstreams,
950971
}
951972
// Correct root if definitely pointing to a file
952-
if fserr == fs.ErrorIsFile {
973+
if errors.Is(fserr, fs.ErrorIsFile) {
953974
f.root = path.Dir(f.root)
954975
if f.root == "." || f.root == "/" {
955976
f.root = ""

0 commit comments

Comments
 (0)