Skip to content

Commit 40c5f56

Browse files
acabarbayeacabarbaye
committed
✨ [artefacts] Extend artefact download to retain the tree structure of the artefacts
1 parent 402057d commit 40c5f56

File tree

9 files changed

+272
-39
lines changed

9 files changed

+272
-39
lines changed

‎changes/20241115120136.bugfix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:arrow_up: upgrade dependencies

‎changes/20241115141349.feature‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:sparkles: `[artefacts]` Extend artefact download to retain the tree structure of the artefacts

‎utils/artefacts/artefacts.go‎

Lines changed: 71 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"net/url"
1313
"os"
1414
"path/filepath"
15+
"strings"
1516

1617
"github.com/ARM-software/embedded-development-services-client-utils/utils/api"
1718
paginationUtils "github.com/ARM-software/embedded-development-services-client-utils/utils/pagination"
@@ -24,6 +25,8 @@ import (
2425
"github.com/ARM-software/golang-utils/utils/safeio"
2526
)
2627

28+
const relativePathKey = "Relative Path"
29+
2730
type (
2831
// GetArtefactManagersFirstPageFunc defines the function which can retrieve the first page of artefact managers.
2932
GetArtefactManagersFirstPageFunc = func(ctx context.Context, job string) (*client.ArtefactManagerCollection, *http.Response, error)
@@ -35,6 +38,45 @@ type (
3538
GetArtefactContentFunc = func(ctx context.Context, job, artefactID string) (*os.File, *http.Response, error)
3639
)
3740

41+
func determineArtefactDestination(outputDir string, maintainTree bool, item *client.ArtefactManagerItem) (artefactFileName string, destinationDir string, err error) {
42+
if item == nil {
43+
err = fmt.Errorf("%w: missing artefact item", commonerrors.ErrUndefined)
44+
return
45+
}
46+
artefactManagerName := item.GetName()
47+
if artefactManagerName == "" {
48+
err = fmt.Errorf("%w: missing artefact name", commonerrors.ErrUndefined)
49+
return
50+
}
51+
rawFileName := artefactManagerName
52+
artefactFileName = rawFileName
53+
if item.HasTitle() {
54+
rawFileName = item.GetTitle()
55+
}
56+
artefactFileName = rawFileName
57+
if unescapedName, err := url.PathUnescape(rawFileName); err == nil {
58+
artefactFileName = unescapedName
59+
}
60+
destinationDir = filepath.Clean(outputDir)
61+
if !maintainTree {
62+
return
63+
}
64+
65+
if item.HasExtraMetadata() {
66+
m := item.GetExtraMetadata()
67+
treePath, ok := m[relativePathKey]
68+
if !ok {
69+
return
70+
}
71+
treePath = strings.TrimSpace(treePath)
72+
if strings.HasSuffix(treePath, rawFileName) || strings.HasSuffix(treePath, artefactFileName) {
73+
treePath = filepath.Dir(treePath)
74+
}
75+
destinationDir = filepath.Clean(filepath.Join(outputDir, treePath))
76+
}
77+
return
78+
}
79+
3880
type ArtefactManager struct {
3981
getArtefactManagerFunc GetArtefactManagerFunc
4082
getArtefactContentFunc GetArtefactContentFunc
@@ -51,8 +93,11 @@ func NewArtefactManager(getArtefactManagersFirstPage GetArtefactManagersFirstPag
5193
getArtefactManagersFollowLinkFunc: getArtefactsManagersPage,
5294
}
5395
}
54-
5596
func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName string, outputDirectory string, artefactManager *client.ArtefactManagerItem) (err error) {
97+
return m.DownloadJobArtefactWithTree(ctx, jobName, false, outputDirectory, artefactManager)
98+
}
99+
100+
func (m *ArtefactManager) DownloadJobArtefactWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManager *client.ArtefactManagerItem) (err error) {
56101
err = parallelisation.DetermineContextError(ctx)
57102
if err != nil {
58103
return
@@ -83,14 +128,6 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
83128
return
84129
}
85130

86-
artefactFilename := artefactManagerName
87-
if artefactManager.HasTitle() {
88-
artefactFilename = artefactManager.GetTitle()
89-
}
90-
if unescapedName, err := url.PathUnescape(artefactFilename); err == nil {
91-
artefactFilename = unescapedName
92-
}
93-
94131
expectedSizePtr, ok := artefactManager.GetSizeOk()
95132
if !ok {
96133
err = fmt.Errorf("%w: could not fetch artefact's size from artefact's manager [%v]", commonerrors.ErrUndefined, artefactManagerName)
@@ -105,6 +142,16 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
105142
}
106143
expectedHash := *expectedHashPtr
107144

145+
artefactFilename, artefactDestDir, err := determineArtefactDestination(outputDirectory, maintainTreeLocation, artefactManager)
146+
if err != nil {
147+
return
148+
}
149+
err = filesystem.MkDir(artefactDestDir)
150+
if err != nil {
151+
err = fmt.Errorf("%w: failed creating the output directory [%v] for job artefact: %v", commonerrors.ErrUnexpected, artefactDestDir, err.Error())
152+
return
153+
}
154+
108155
artefact, resp, apierr := m.getArtefactContentFunc(ctx, jobName, artefactManagerName)
109156
defer func() {
110157
if resp != nil {
@@ -120,7 +167,7 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
120167
return
121168
}
122169

123-
destination, err := filesystem.CreateFile(filepath.Join(outputDirectory, artefactFilename))
170+
destination, err := filesystem.CreateFile(filepath.Join(artefactDestDir, artefactFilename))
124171
if err != nil {
125172
err = fmt.Errorf("%w: could not create a location to store generated artefact [%v]: %v", commonerrors.ErrUnexpected, artefactFilename, err.Error())
126173
return
@@ -161,8 +208,11 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
161208
return
162209

163210
}
211+
func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobName string, outputDirectory string, artefactManagerItemLink *client.HalLinkData) error {
212+
return m.DownloadJobArtefactFromLinkWithTree(ctx, jobName, false, outputDirectory, artefactManagerItemLink)
213+
}
164214

165-
func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobName string, outputDirectory string, artefactManagerItem *client.HalLinkData) (err error) {
215+
func (m *ArtefactManager) DownloadJobArtefactFromLinkWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManagerItemLink *client.HalLinkData) (err error) {
166216
err = parallelisation.DetermineContextError(ctx)
167217
if err != nil {
168218
return
@@ -171,12 +221,12 @@ func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobNa
171221
err = fmt.Errorf("%w: function to retrieve an artefact manager was not properly defined", commonerrors.ErrUndefined)
172222
return
173223
}
174-
if artefactManagerItem == nil {
224+
if artefactManagerItemLink == nil {
175225
err = fmt.Errorf("%w: missing artefact link", commonerrors.ErrUndefined)
176226
return
177227
}
178228

179-
artefactManagerName := artefactManagerItem.GetName()
229+
artefactManagerName := artefactManagerItemLink.GetName()
180230
artefactManager, resp, apierr := m.getArtefactManagerFunc(ctx, jobName, artefactManagerName)
181231
defer func() {
182232
if resp != nil {
@@ -190,7 +240,7 @@ func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobNa
190240
if resp != nil {
191241
_ = resp.Body.Close()
192242
}
193-
err = m.DownloadJobArtefact(ctx, jobName, outputDirectory, artefactManager)
243+
err = m.DownloadJobArtefactWithTree(ctx, jobName, maintainTreeLocation, outputDirectory, artefactManager)
194244
return
195245
}
196246

@@ -263,7 +313,11 @@ func (m *ArtefactManager) fetchJobArtefactsNextPage(ctx context.Context, current
263313
return
264314
}
265315

266-
func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName string, outputDirectory string) (err error) {
316+
func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName string, outputDirectory string) error {
317+
return m.DownloadAllJobArtefactsWithTree(ctx, jobName, false, outputDirectory)
318+
}
319+
320+
func (m *ArtefactManager) DownloadAllJobArtefactsWithTree(ctx context.Context, jobName string, maintainTreeStructure bool, outputDirectory string) (err error) {
267321
err = parallelisation.DetermineContextError(ctx)
268322
if err != nil {
269323
return
@@ -290,7 +344,7 @@ func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName s
290344
}
291345
artefactLink, ok := item.(*client.HalLinkData)
292346
if ok {
293-
subErr = m.DownloadJobArtefactFromLink(ctx, jobName, outputDirectory, artefactLink)
347+
subErr = m.DownloadJobArtefactFromLinkWithTree(ctx, jobName, maintainTreeStructure, outputDirectory, artefactLink)
294348
if subErr != nil {
295349
err = subErr
296350
return
@@ -299,7 +353,7 @@ func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName s
299353
} else {
300354
artefactManager, ok := item.(*client.ArtefactManagerItem)
301355
if ok {
302-
subErr = m.DownloadJobArtefact(ctx, jobName, outputDirectory, artefactManager)
356+
subErr = m.DownloadJobArtefactWithTree(ctx, jobName, maintainTreeStructure, outputDirectory, artefactManager)
303357
if subErr != nil {
304358
err = subErr
305359
return

‎utils/artefacts/artefacts_test.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"net/http"
1212
"os"
1313
"path/filepath"
14+
"strings"
1415
"testing"
1516
"time"
1617

@@ -164,6 +165,113 @@ func newTestArtefactManagerWithEmbeddedResources(t *testing.T, tmpDir, artefactC
164165
return newTestArtefactManager(t, tmpDir, artefactContent, false)
165166
}
166167

168+
func TestDetermineDestination(t *testing.T) {
169+
outputDir := strings.ReplaceAll(faker.Sentence(), " ", "//") + " "
170+
cleanedOutputDir := filepath.Clean(outputDir)
171+
172+
tests := []struct {
173+
item client.ArtefactManagerItem
174+
maintainTree bool
175+
outputDir string
176+
expectedFileName string
177+
expectedDir string
178+
}{
179+
{
180+
item: client.ArtefactManagerItem{
181+
ExtraMetadata: nil,
182+
Name: faker.Name(),
183+
Size: nil,
184+
Title: *client.NewNullableString(field.ToOptionalString("test.j")),
185+
},
186+
maintainTree: false,
187+
outputDir: outputDir,
188+
expectedFileName: "test.j",
189+
expectedDir: cleanedOutputDir,
190+
},
191+
{
192+
item: client.ArtefactManagerItem{
193+
ExtraMetadata: nil,
194+
Name: faker.Name(),
195+
Size: nil,
196+
Title: *client.NewNullableString(field.ToOptionalString("test.j")),
197+
},
198+
maintainTree: true,
199+
outputDir: outputDir,
200+
expectedFileName: "test.j",
201+
expectedDir: cleanedOutputDir,
202+
},
203+
{
204+
item: client.ArtefactManagerItem{
205+
ExtraMetadata: nil,
206+
Name: faker.Name(),
207+
Size: nil,
208+
Title: *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
209+
},
210+
maintainTree: true,
211+
outputDir: outputDir,
212+
expectedFileName: "cool+blog&about,stuff.yep",
213+
expectedDir: cleanedOutputDir,
214+
},
215+
{
216+
item: client.ArtefactManagerItem{
217+
ExtraMetadata: &map[string]string{faker.Name(): faker.Sentence()},
218+
Name: faker.Name(),
219+
Size: nil,
220+
Title: *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
221+
},
222+
maintainTree: true,
223+
outputDir: outputDir,
224+
expectedFileName: "cool+blog&about,stuff.yep",
225+
expectedDir: cleanedOutputDir,
226+
},
227+
{
228+
item: client.ArtefactManagerItem{
229+
ExtraMetadata: &map[string]string{relativePathKey: " test/1 "},
230+
Name: faker.Name(),
231+
Size: nil,
232+
Title: *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
233+
},
234+
maintainTree: true,
235+
outputDir: outputDir,
236+
expectedFileName: "cool+blog&about,stuff.yep",
237+
expectedDir: filepath.Join(cleanedOutputDir, "test", "1"),
238+
},
239+
{
240+
item: client.ArtefactManagerItem{
241+
ExtraMetadata: &map[string]string{relativePathKey: " test/1/cool+blog&about,stuff.yep "},
242+
Name: faker.Name(),
243+
Size: nil,
244+
Title: *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
245+
},
246+
maintainTree: true,
247+
outputDir: outputDir,
248+
expectedFileName: "cool+blog&about,stuff.yep",
249+
expectedDir: filepath.Join(cleanedOutputDir, "test", "1"),
250+
},
251+
{
252+
item: client.ArtefactManagerItem{
253+
ExtraMetadata: &map[string]string{relativePathKey: " test/1/cool+blog&about%2Cstuff.yep "},
254+
Name: faker.Name(),
255+
Size: nil,
256+
Title: *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
257+
},
258+
maintainTree: true,
259+
outputDir: outputDir,
260+
expectedFileName: "cool+blog&about,stuff.yep",
261+
expectedDir: filepath.Join(cleanedOutputDir, "test", "1"),
262+
},
263+
}
264+
for i := range tests {
265+
test := tests[i]
266+
t.Run(fmt.Sprintf("%d_%s", i, test.expectedFileName), func(t *testing.T) {
267+
fileName, fileDest, err := determineArtefactDestination(test.outputDir, test.maintainTree, &test.item)
268+
require.NoError(t, err)
269+
assert.Equal(t, test.expectedFileName, fileName)
270+
assert.Equal(t, test.expectedDir, fileDest)
271+
})
272+
}
273+
}
274+
167275
func TestArtefactDownload(t *testing.T) {
168276
t.Run("Happy download artefact", func(t *testing.T) {
169277
tmpDir, err := filesystem.TempDirInTempDir("test-artefact-")
@@ -184,7 +292,25 @@ func TestArtefactDownload(t *testing.T) {
184292
require.NoError(t, err)
185293
assert.Equal(t, expectedContents, actualContents)
186294
})
295+
t.Run("Happy download artefact and keep tree", func(t *testing.T) {
296+
tmpDir, err := filesystem.TempDirInTempDir("test-artefact-with-tree-")
297+
require.NoError(t, err)
298+
defer func() { _ = filesystem.Rm(tmpDir) }()
299+
m, a := newTestArtefactManagerWithEmbeddedResources(t, tmpDir, faker.Sentence())
300+
301+
out := t.TempDir()
302+
err = m.DownloadJobArtefactFromLinkWithTree(context.Background(), faker.Word(), true, out, &client.HalLinkData{
303+
Name: &a.name,
304+
})
305+
require.NoError(t, err)
187306

307+
require.FileExists(t, filepath.Join(out, a.name))
308+
expectedContents, err := filesystem.ReadFile(a.path)
309+
require.NoError(t, err)
310+
actualContents, err := filesystem.ReadFile(filepath.Join(out, a.name))
311+
require.NoError(t, err)
312+
assert.Equal(t, expectedContents, actualContents)
313+
})
188314
t.Run("Happy list artefacts links", func(t *testing.T) {
189315
tmpDir, err := filesystem.TempDirInTempDir("test-artefact-")
190316
require.NoError(t, err)

‎utils/artefacts/interface.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,21 @@ import (
1515
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/embedded-development-services-client-utils/utils/$GOPACKAGE IArtefactManager
1616

1717
type IArtefactManager interface {
18-
// DownloadJobArtefactFromLink downloads a specific artefact into the output directory from a particular link.
18+
// DownloadJobArtefactFromLink downloads a specific artefact into the output directory from a particular link. The artefact will be placed at the root of the output directory.
1919
DownloadJobArtefactFromLink(ctx context.Context, jobName string, outputDirectory string, artefactManagerItemLink *client.HalLinkData) error
20-
// DownloadJobArtefact downloads a specific artefact into the output directory.
20+
// DownloadJobArtefactFromLinkWithTree downloads a specific artefact into the output directory from a particular link.
21+
// maintainTreeLocation specifies whether the artefact will be placed in a tree structure or if it will be flat.
22+
DownloadJobArtefactFromLinkWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManagerItemLink *client.HalLinkData) error
23+
// DownloadJobArtefact downloads a specific artefact into the output directory. The artefact will be placed at the root of the output directory.
2124
DownloadJobArtefact(ctx context.Context, jobName string, outputDirectory string, artefactManager *client.ArtefactManagerItem) error
25+
// DownloadJobArtefactWithTree downloads a specific artefact into the output directory.
26+
// maintainTreeLocation specifies whether the artefact will be placed in a tree structure or if it will be flat.
27+
DownloadJobArtefactWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManager *client.ArtefactManagerItem) error
2228
// ListJobArtefacts lists all artefact managers associated with a particular job.
2329
ListJobArtefacts(ctx context.Context, jobName string) (pagination.IPaginatorAndPageFetcher, error)
24-
// DownloadAllJobArtefacts downloads all the artefacts produced for a particular job and puts them in an output directory.
30+
// DownloadAllJobArtefacts downloads all the artefacts produced for a particular job and puts them in an output directory as a flat list.
2531
DownloadAllJobArtefacts(ctx context.Context, jobName string, outputDirectory string) error
32+
// DownloadAllJobArtefactsWithTree downloads all the artefacts produced for a particular job and puts them in an output directory.
33+
// maintainTreeStructure specifies whether to keep the tree structure of the artefacts or not in the output directory.
34+
DownloadAllJobArtefactsWithTree(ctx context.Context, jobName string, maintainTreeStructure bool, outputDirectory string) error
2635
}

‎utils/go.mod‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@ module github.com/ARM-software/embedded-development-services-client-utils/utils
33
go 1.23
44

55
require (
6-
github.com/ARM-software/embedded-development-services-client/client v1.35.3
7-
github.com/ARM-software/golang-utils/utils v1.73.2
6+
github.com/ARM-software/embedded-development-services-client/client v1.36.1
7+
github.com/ARM-software/golang-utils/utils v1.74.1
88
github.com/go-faker/faker/v4 v4.5.0
99
github.com/go-logr/logr v1.4.2
1010
github.com/golang/mock v1.6.0
1111
github.com/stretchr/testify v1.9.0
1212
go.uber.org/atomic v1.11.0
1313
go.uber.org/goleak v1.3.0
14-
golang.org/x/sync v0.8.0
14+
golang.org/x/sync v0.9.0
1515
)
1616

1717
require (
@@ -68,10 +68,10 @@ require (
6868
github.com/zailic/slogr v0.0.2-alpha // indirect
6969
go.uber.org/multierr v1.10.0 // indirect
7070
go.uber.org/zap v1.27.0 // indirect
71-
golang.org/x/crypto v0.28.0 // indirect
71+
golang.org/x/crypto v0.29.0 // indirect
7272
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
73-
golang.org/x/sys v0.26.0 // indirect
74-
golang.org/x/text v0.19.0 // indirect
73+
golang.org/x/sys v0.27.0 // indirect
74+
golang.org/x/text v0.20.0 // indirect
7575
gopkg.in/ini.v1 v1.67.0 // indirect
7676
gopkg.in/yaml.v3 v3.0.1 // indirect
7777
)

0 commit comments

Comments
 (0)