Skip to content

Commit 8f7c6f1

Browse files
Gerrit91Gerrit91
authored
Cleanup empty directories. (#9)
1 parent 305fa43 commit 8f7c6f1

File tree

2 files changed

+194
-2
lines changed

2 files changed

+194
-2
lines changed

‎cmd/internal/sync/syncer.go‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package sync
22

33
import (
44
"context"
5+
56
// nolint
67
"crypto/md5"
78
"fmt"
@@ -90,14 +91,19 @@ func (s *Syncer) Sync(rootPath string, entitiesToSync api.CacheEntities) error {
9091
}
9192
}
9293

94+
err = cleanEmptyDirs(s.fs, rootPath)
95+
if err != nil {
96+
return errors.Wrap(err, "error cleaning up empty directories")
97+
}
98+
9399
return nil
94100
}
95101

96102
func currentFileIndex(fs afero.Fs, rootPath string) (api.CacheEntities, error) {
97103
var result api.CacheEntities
98104
err := afero.Walk(fs, rootPath, func(p string, info os.FileInfo, innerErr error) error {
99105
if innerErr != nil {
100-
return errors.Wrap(innerErr, "error while walking through cache root")
106+
return errors.Wrap(innerErr, fmt.Sprintf("error while walking through root path %s", rootPath))
101107
}
102108

103109
if info.IsDir() {
@@ -308,3 +314,57 @@ func (s *Syncer) printSyncPlan(remove api.CacheEntities, keep []api.CacheEntity,
308314
}
309315
table.Render()
310316
}
317+
318+
func cleanEmptyDirs(fs afero.Fs, rootPath string) error {
319+
files, err := afero.ReadDir(fs, rootPath)
320+
if err != nil {
321+
return err
322+
}
323+
324+
for _, info := range files {
325+
if !info.IsDir() {
326+
continue
327+
}
328+
329+
err = recurseCleanEmptyDirs(fs, path.Join(rootPath, info.Name()))
330+
if err != nil {
331+
return err
332+
}
333+
}
334+
335+
return nil
336+
}
337+
338+
func recurseCleanEmptyDirs(fs afero.Fs, p string) error {
339+
files, err := afero.ReadDir(fs, p)
340+
if err != nil {
341+
return err
342+
}
343+
344+
for _, info := range files {
345+
if !info.IsDir() {
346+
continue
347+
}
348+
349+
nested := path.Join(p, info.Name())
350+
err = recurseCleanEmptyDirs(fs, nested)
351+
if err != nil {
352+
return err
353+
}
354+
}
355+
356+
// re-read files because directories could delete themselves in first loop
357+
files, err = afero.ReadDir(fs, p)
358+
if err != nil {
359+
return err
360+
}
361+
362+
if len(files) == 0 {
363+
err = fs.Remove(p)
364+
if err != nil {
365+
return err
366+
}
367+
}
368+
369+
return nil
370+
}

‎cmd/internal/sync/syncer_test.go‎

Lines changed: 133 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/google/go-cmp/cmp/cmpopts"
2323
"github.com/metal-stack/metal-image-cache-sync/pkg/api"
2424
"github.com/spf13/afero"
25+
"github.com/stretchr/testify/assert"
2526
"github.com/stretchr/testify/require"
2627
"go.uber.org/zap/zaptest"
2728
)
@@ -94,14 +95,18 @@ func Test_currentFileIndex(t *testing.T) {
9495
}
9596

9697
func createTestFile(t *testing.T, fs afero.Fs, p string) {
97-
require.Nil(t, fs.MkdirAll(path.Base(p), 0755))
98+
createTestDir(t, fs, path.Base(p))
9899
f, err := fs.Create(p)
99100
require.Nil(t, err)
100101
defer f.Close()
101102
_, err = f.WriteString("Test")
102103
require.Nil(t, err)
103104
}
104105

106+
func createTestDir(t *testing.T, fs afero.Fs, p string) {
107+
require.Nil(t, fs.MkdirAll(p, 0755))
108+
}
109+
105110
func dlLoggingSvc(data []byte) (*s3.S3, *[]string, *[]string) {
106111
var m sync.Mutex
107112
names := []string{}
@@ -346,3 +351,130 @@ func TestSyncer_defineImageDiff(t *testing.T) {
346351
func strPtr(s string) *string {
347352
return &s
348353
}
354+
355+
func Test_cleanEmptyDirs(t *testing.T) {
356+
tests := []struct {
357+
name string
358+
fsModFunc func(t *testing.T, fs afero.Fs)
359+
fsCheckFunc func(t *testing.T, fs afero.Fs)
360+
wantErr error
361+
}{
362+
{
363+
name: "no directory contents, nothing happens",
364+
fsModFunc: nil,
365+
wantErr: nil,
366+
},
367+
{
368+
name: "flat deletion",
369+
fsModFunc: func(t *testing.T, fs afero.Fs) {
370+
createTestDir(t, fs, cacheRoot+"/ubuntu")
371+
},
372+
fsCheckFunc: func(t *testing.T, fs afero.Fs) {
373+
exists, err := afero.Exists(fs, cacheRoot+"/ubuntu")
374+
assert.NoError(t, err)
375+
assert.False(t, exists, "dir still exists")
376+
},
377+
wantErr: nil,
378+
},
379+
{
380+
name: "recursive deletion 1",
381+
fsModFunc: func(t *testing.T, fs afero.Fs) {
382+
createTestDir(t, fs, cacheRoot+"/ubuntu/20.10/20201027")
383+
},
384+
fsCheckFunc: func(t *testing.T, fs afero.Fs) {
385+
exists, err := afero.Exists(fs, cacheRoot+"/ubuntu/20.10/20201027")
386+
assert.NoError(t, err)
387+
assert.False(t, exists, "dir still exists")
388+
389+
exists, err = afero.Exists(fs, cacheRoot+"/ubuntu/20.10")
390+
assert.NoError(t, err)
391+
assert.False(t, exists, "dir still exists")
392+
393+
exists, err = afero.Exists(fs, cacheRoot+"/ubuntu")
394+
assert.NoError(t, err)
395+
assert.False(t, exists, "dir still exists")
396+
},
397+
wantErr: nil,
398+
},
399+
{
400+
name: "recursive deletion 2",
401+
fsModFunc: func(t *testing.T, fs afero.Fs) {
402+
createTestFile(t, fs, cacheRoot+"/ubuntu/20.04/20201028/img.tar.lz4")
403+
createTestDir(t, fs, cacheRoot+"/ubuntu/20.10/20201027")
404+
},
405+
fsCheckFunc: func(t *testing.T, fs afero.Fs) {
406+
exists, err := afero.Exists(fs, cacheRoot+"/ubuntu/20.10/20201027")
407+
assert.NoError(t, err)
408+
assert.False(t, exists, "dir still exists")
409+
410+
exists, err = afero.Exists(fs, cacheRoot+"/ubuntu/20.10")
411+
assert.NoError(t, err)
412+
assert.False(t, exists, "dir still exists")
413+
414+
exists, err = afero.Exists(fs, cacheRoot+"/ubuntu")
415+
assert.NoError(t, err)
416+
assert.True(t, exists, "dir was deleted")
417+
},
418+
wantErr: nil,
419+
},
420+
{
421+
name: "kind of realistic scenario",
422+
423+
fsModFunc: func(t *testing.T, fs afero.Fs) {
424+
createTestDir(t, fs, cacheRoot+"/boot/metal-hammer/releases/download/v0.8.0")
425+
createTestFile(t, fs, cacheRoot+"/boot/metal-hammer/pull-requests/pr-title/metal-hammer-initrd.img.lz4")
426+
createTestFile(t, fs, cacheRoot+"/boot/metal-hammer/pull-requests/pr-title/metal-hammer-initrd.img.lz4.md5")
427+
createTestFile(t, fs, cacheRoot+"/ubuntu/20.10/20201026/img.tar.lz4")
428+
createTestFile(t, fs, cacheRoot+"/ubuntu/20.10/20201026/img.tar.lz4.md5")
429+
createTestDir(t, fs, cacheRoot+"/firewall/2.0/20210131")
430+
createTestDir(t, fs, cacheRoot+"/firewall/2.0/20210207")
431+
createTestFile(t, fs, cacheRoot+"/firewall/2.0/20210304/img.tar.lz4")
432+
createTestFile(t, fs, cacheRoot+"/firewall/2.0/20210304/img.tar.lz4.md5")
433+
},
434+
fsCheckFunc: func(t *testing.T, fs afero.Fs) {
435+
for _, subPath := range []string{
436+
"/boot/metal-hammer/releases",
437+
"/firewall/2.0.20210131",
438+
"/firewall/2.0.20210207",
439+
} {
440+
exists, err := afero.Exists(fs, cacheRoot+subPath)
441+
assert.NoError(t, err)
442+
assert.False(t, exists, "dir still exists")
443+
}
444+
445+
for _, subPath := range []string{
446+
"/boot/metal-hammer/pull-requests/pr-title/metal-hammer-initrd.img.lz4",
447+
"/boot/metal-hammer/pull-requests/pr-title/metal-hammer-initrd.img.lz4.md5",
448+
"/ubuntu/20.10/20201026/img.tar.lz4",
449+
"/ubuntu/20.10/20201026/img.tar.lz4.md5",
450+
"/firewall/2.0/20210304/img.tar.lz4",
451+
"/firewall/2.0/20210304/img.tar.lz4.md5",
452+
} {
453+
exists, err := afero.Exists(fs, cacheRoot+subPath)
454+
assert.NoError(t, err)
455+
assert.True(t, exists, "dir was deleted")
456+
}
457+
},
458+
wantErr: nil,
459+
},
460+
}
461+
for _, tt := range tests {
462+
tt := tt
463+
t.Run(tt.name, func(t *testing.T) {
464+
fs := afero.NewMemMapFs()
465+
require.Nil(t, fs.MkdirAll(cacheRoot, 0755))
466+
if tt.fsModFunc != nil {
467+
tt.fsModFunc(t, fs)
468+
}
469+
470+
err := cleanEmptyDirs(fs, cacheRoot)
471+
if diff := cmp.Diff(err, tt.wantErr); diff != "" {
472+
t.Errorf("cleanEmptyDirs() diff = %v", diff)
473+
}
474+
475+
if tt.fsCheckFunc != nil {
476+
tt.fsCheckFunc(t, fs)
477+
}
478+
})
479+
}
480+
}

0 commit comments

Comments
 (0)