✨ [artefacts] Extend artefact download to retain the tree structure of the artefacts (#88)

<!--
Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors.
All rights reserved.
SPDX-License-Identifier: Proprietary
-->
### Description
- This is so that artefacts tree is automatically reproduced in client
machine



### Test Coverage

<!--
Please put an `x` in the correct box e.g. `[x]` to indicate the testing
coverage of this change.
-->

- [x]  This change is covered by existing or additional automated tests.
- [ ] Manual testing has been performed (and evidence provided) as
automated testing was not feasible.
- [ ] Additional tests are not required for this change (e.g.
documentation update).

Author: acabarbaye

changes/20241115120136.bugfix

@@ -0,0 +1 @@
+:arrow_up: upgrade dependencies

changes/20241115141349.feature

@@ -0,0 +1 @@
+:sparkles: `[artefacts]` Extend artefact download to retain the tree structure of the artefacts

utils/artefacts/artefacts.go

@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/ARM-software/embedded-development-services-client-utils/utils/api"
 	paginationUtils "github.com/ARM-software/embedded-development-services-client-utils/utils/pagination"
@@ -24,6 +25,8 @@ import (
 	"github.com/ARM-software/golang-utils/utils/safeio"
 )
 
+const relativePathKey = "Relative Path"
+
 type (
 	// GetArtefactManagersFirstPageFunc defines the function which can retrieve the first page of artefact managers.
 	GetArtefactManagersFirstPageFunc = func(ctx context.Context, job string) (*client.ArtefactManagerCollection, *http.Response, error)
@@ -35,6 +38,44 @@ type (
 	GetArtefactContentFunc = func(ctx context.Context, job, artefactID string) (*os.File, *http.Response, error)
 )
 
+func determineArtefactDestination(outputDir string, maintainTree bool, item *client.ArtefactManagerItem) (artefactFileName string, destinationDir string, err error) {
+	if item == nil {
+		err = fmt.Errorf("%w: missing artefact item", commonerrors.ErrUndefined)
+		return
+	}
+	artefactManagerName := item.GetName()
+	if artefactManagerName == "" {
+		err = fmt.Errorf("%w: missing artefact name", commonerrors.ErrUndefined)
+		return
+	}
+	rawFileName := artefactManagerName
+	if item.HasTitle() {
+		rawFileName = item.GetTitle()
+	}
+	artefactFileName = rawFileName
+	if unescapedName, err := url.PathUnescape(rawFileName); err == nil {
+		artefactFileName = unescapedName
+	}
+	destinationDir = filepath.Clean(outputDir)
+	if !maintainTree {
+		return
+	}
+
+	if item.HasExtraMetadata() {
+		m := item.GetExtraMetadata()
+		treePath, ok := m[relativePathKey]
+		if !ok {
+			return
+		}
+		treePath = strings.TrimSpace(treePath)
+		if strings.HasSuffix(treePath, rawFileName) || strings.HasSuffix(treePath, artefactFileName) {
+			treePath = filepath.Dir(treePath)
+		}
+		destinationDir = filepath.Clean(filepath.Join(outputDir, treePath))
+	}
+	return
+}
+
 type ArtefactManager struct {
 	getArtefactManagerFunc            GetArtefactManagerFunc
 	getArtefactContentFunc            GetArtefactContentFunc
@@ -51,8 +92,11 @@ func NewArtefactManager(getArtefactManagersFirstPage GetArtefactManagersFirstPag
 		getArtefactManagersFollowLinkFunc: getArtefactsManagersPage,
 	}
 }
-
 func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName string, outputDirectory string, artefactManager *client.ArtefactManagerItem) (err error) {
+	return m.DownloadJobArtefactWithTree(ctx, jobName, false, outputDirectory, artefactManager)
+}
+
+func (m *ArtefactManager) DownloadJobArtefactWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManager *client.ArtefactManagerItem) (err error) {
 	err = parallelisation.DetermineContextError(ctx)
 	if err != nil {
 		return
@@ -83,14 +127,6 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
 		return
 	}
 
-	artefactFilename := artefactManagerName
-	if artefactManager.HasTitle() {
-		artefactFilename = artefactManager.GetTitle()
-	}
-	if unescapedName, err := url.PathUnescape(artefactFilename); err == nil {
-		artefactFilename = unescapedName
-	}
-
 	expectedSizePtr, ok := artefactManager.GetSizeOk()
 	if !ok {
 		err = fmt.Errorf("%w: could not fetch artefact's size from artefact's manager [%v]", commonerrors.ErrUndefined, artefactManagerName)
@@ -105,6 +141,16 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
 	}
 	expectedHash := *expectedHashPtr
 
+	artefactFilename, artefactDestDir, err := determineArtefactDestination(outputDirectory, maintainTreeLocation, artefactManager)
+	if err != nil {
+		return
+	}
+	err = filesystem.MkDir(artefactDestDir)
+	if err != nil {
+		err = fmt.Errorf("%w: failed creating the output directory [%v] for job artefact: %v", commonerrors.ErrUnexpected, artefactDestDir, err.Error())
+		return
+	}
+
 	artefact, resp, apierr := m.getArtefactContentFunc(ctx, jobName, artefactManagerName)
 	defer func() {
 		if resp != nil {
@@ -120,7 +166,7 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
 		return
 	}
 
-	destination, err := filesystem.CreateFile(filepath.Join(outputDirectory, artefactFilename))
+	destination, err := filesystem.CreateFile(filepath.Join(artefactDestDir, artefactFilename))
 	if err != nil {
 		err = fmt.Errorf("%w: could not create a location to store generated artefact [%v]: %v", commonerrors.ErrUnexpected, artefactFilename, err.Error())
 		return
@@ -161,8 +207,11 @@ func (m *ArtefactManager) DownloadJobArtefact(ctx context.Context, jobName strin
 	return
 
 }
+func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobName string, outputDirectory string, artefactManagerItemLink *client.HalLinkData) error {
+	return m.DownloadJobArtefactFromLinkWithTree(ctx, jobName, false, outputDirectory, artefactManagerItemLink)
+}
 
-func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobName string, outputDirectory string, artefactManagerItem *client.HalLinkData) (err error) {
+func (m *ArtefactManager) DownloadJobArtefactFromLinkWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManagerItemLink *client.HalLinkData) (err error) {
 	err = parallelisation.DetermineContextError(ctx)
 	if err != nil {
 		return
@@ -171,12 +220,12 @@ func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobNa
 		err = fmt.Errorf("%w: function to retrieve an artefact manager was not properly defined", commonerrors.ErrUndefined)
 		return
 	}
-	if artefactManagerItem == nil {
+	if artefactManagerItemLink == nil {
 		err = fmt.Errorf("%w: missing artefact link", commonerrors.ErrUndefined)
 		return
 	}
 
-	artefactManagerName := artefactManagerItem.GetName()
+	artefactManagerName := artefactManagerItemLink.GetName()
 	artefactManager, resp, apierr := m.getArtefactManagerFunc(ctx, jobName, artefactManagerName)
 	defer func() {
 		if resp != nil {
@@ -190,7 +239,7 @@ func (m *ArtefactManager) DownloadJobArtefactFromLink(ctx context.Context, jobNa
 	if resp != nil {
 		_ = resp.Body.Close()
 	}
-	err = m.DownloadJobArtefact(ctx, jobName, outputDirectory, artefactManager)
+	err = m.DownloadJobArtefactWithTree(ctx, jobName, maintainTreeLocation, outputDirectory, artefactManager)
 	return
 }
 
@@ -263,7 +312,11 @@ func (m *ArtefactManager) fetchJobArtefactsNextPage(ctx context.Context, current
 	return
 }
 
-func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName string, outputDirectory string) (err error) {
+func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName string, outputDirectory string) error {
+	return m.DownloadAllJobArtefactsWithTree(ctx, jobName, false, outputDirectory)
+}
+
+func (m *ArtefactManager) DownloadAllJobArtefactsWithTree(ctx context.Context, jobName string, maintainTreeStructure bool, outputDirectory string) (err error) {
 	err = parallelisation.DetermineContextError(ctx)
 	if err != nil {
 		return
@@ -290,7 +343,7 @@ func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName s
 		}
 		artefactLink, ok := item.(*client.HalLinkData)
 		if ok {
-			subErr = m.DownloadJobArtefactFromLink(ctx, jobName, outputDirectory, artefactLink)
+			subErr = m.DownloadJobArtefactFromLinkWithTree(ctx, jobName, maintainTreeStructure, outputDirectory, artefactLink)
 			if subErr != nil {
 				err = subErr
 				return
@@ -299,7 +352,7 @@ func (m *ArtefactManager) DownloadAllJobArtefacts(ctx context.Context, jobName s
 		} else {
 			artefactManager, ok := item.(*client.ArtefactManagerItem)
 			if ok {
-				subErr = m.DownloadJobArtefact(ctx, jobName, outputDirectory, artefactManager)
+				subErr = m.DownloadJobArtefactWithTree(ctx, jobName, maintainTreeStructure, outputDirectory, artefactManager)
 				if subErr != nil {
 					err = subErr
 					return

utils/artefacts/artefacts_test.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -164,6 +165,113 @@ func newTestArtefactManagerWithEmbeddedResources(t *testing.T, tmpDir, artefactC
 	return newTestArtefactManager(t, tmpDir, artefactContent, false)
 }
 
+func TestDetermineDestination(t *testing.T) {
+	outputDir := strings.ReplaceAll(faker.Sentence(), " ", "//") + "                  "
+	cleanedOutputDir := filepath.Clean(outputDir)
+
+	tests := []struct {
+		item             client.ArtefactManagerItem
+		maintainTree     bool
+		outputDir        string
+		expectedFileName string
+		expectedDir      string
+	}{
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: nil,
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("test.j")),
+			},
+			maintainTree:     false,
+			outputDir:        outputDir,
+			expectedFileName: "test.j",
+			expectedDir:      cleanedOutputDir,
+		},
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: nil,
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("test.j")),
+			},
+			maintainTree:     true,
+			outputDir:        outputDir,
+			expectedFileName: "test.j",
+			expectedDir:      cleanedOutputDir,
+		},
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: nil,
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
+			},
+			maintainTree:     true,
+			outputDir:        outputDir,
+			expectedFileName: "cool+blog&about,stuff.yep",
+			expectedDir:      cleanedOutputDir,
+		},
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: &map[string]string{faker.Name(): faker.Sentence()},
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
+			},
+			maintainTree:     true,
+			outputDir:        outputDir,
+			expectedFileName: "cool+blog&about,stuff.yep",
+			expectedDir:      cleanedOutputDir,
+		},
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: &map[string]string{relativePathKey: "        test/1     "},
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
+			},
+			maintainTree:     true,
+			outputDir:        outputDir,
+			expectedFileName: "cool+blog&about,stuff.yep",
+			expectedDir:      filepath.Join(cleanedOutputDir, "test", "1"),
+		},
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: &map[string]string{relativePathKey: "        test/1/cool+blog&about,stuff.yep     "},
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
+			},
+			maintainTree:     true,
+			outputDir:        outputDir,
+			expectedFileName: "cool+blog&about,stuff.yep",
+			expectedDir:      filepath.Join(cleanedOutputDir, "test", "1"),
+		},
+		{
+			item: client.ArtefactManagerItem{
+				ExtraMetadata: &map[string]string{relativePathKey: "        test/1/cool+blog&about%2Cstuff.yep     "},
+				Name:          faker.Name(),
+				Size:          nil,
+				Title:         *client.NewNullableString(field.ToOptionalString("cool+blog&about%2Cstuff.yep")),
+			},
+			maintainTree:     true,
+			outputDir:        outputDir,
+			expectedFileName: "cool+blog&about,stuff.yep",
+			expectedDir:      filepath.Join(cleanedOutputDir, "test", "1"),
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(fmt.Sprintf("%d_%s", i, test.expectedFileName), func(t *testing.T) {
+			fileName, fileDest, err := determineArtefactDestination(test.outputDir, test.maintainTree, &test.item)
+			require.NoError(t, err)
+			assert.Equal(t, test.expectedFileName, fileName)
+			assert.Equal(t, test.expectedDir, fileDest)
+		})
+	}
+}
+
 func TestArtefactDownload(t *testing.T) {
 	t.Run("Happy download artefact", func(t *testing.T) {
 		tmpDir, err := filesystem.TempDirInTempDir("test-artefact-")
@@ -184,7 +292,25 @@ func TestArtefactDownload(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, expectedContents, actualContents)
 	})
+	t.Run("Happy download artefact and keep tree", func(t *testing.T) {
+		tmpDir, err := filesystem.TempDirInTempDir("test-artefact-with-tree-")
+		require.NoError(t, err)
+		defer func() { _ = filesystem.Rm(tmpDir) }()
+		m, a := newTestArtefactManagerWithEmbeddedResources(t, tmpDir, faker.Sentence())
+
+		out := t.TempDir()
+		err = m.DownloadJobArtefactFromLinkWithTree(context.Background(), faker.Word(), true, out, &client.HalLinkData{
+			Name: &a.name,
+		})
+		require.NoError(t, err)
 
+		require.FileExists(t, filepath.Join(out, a.name))
+		expectedContents, err := filesystem.ReadFile(a.path)
+		require.NoError(t, err)
+		actualContents, err := filesystem.ReadFile(filepath.Join(out, a.name))
+		require.NoError(t, err)
+		assert.Equal(t, expectedContents, actualContents)
+	})
 	t.Run("Happy list artefacts links", func(t *testing.T) {
 		tmpDir, err := filesystem.TempDirInTempDir("test-artefact-")
 		require.NoError(t, err)

utils/artefacts/interface.go

@@ -15,12 +15,21 @@ import (
 //go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/embedded-development-services-client-utils/utils/$GOPACKAGE IArtefactManager
 
 type IArtefactManager interface {
-	// DownloadJobArtefactFromLink downloads a specific artefact into the output directory from a particular link.
+	// DownloadJobArtefactFromLink downloads a specific artefact into the output directory from a particular link. The artefact will be placed at the root of the output directory.
 	DownloadJobArtefactFromLink(ctx context.Context, jobName string, outputDirectory string, artefactManagerItemLink *client.HalLinkData) error
-	// DownloadJobArtefact downloads a specific artefact into the output directory.
+	// DownloadJobArtefactFromLinkWithTree downloads a specific artefact into the output directory from a particular link.
+	// maintainTreeLocation specifies whether the artefact will be placed in a tree structure or if it will be flat.
+	DownloadJobArtefactFromLinkWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManagerItemLink *client.HalLinkData) error
+	// DownloadJobArtefact downloads a specific artefact into the output directory. The artefact will be placed at the root of the output directory.
 	DownloadJobArtefact(ctx context.Context, jobName string, outputDirectory string, artefactManager *client.ArtefactManagerItem) error
+	// DownloadJobArtefactWithTree downloads a specific artefact into the output directory.
+	// maintainTreeLocation specifies whether the artefact will be placed in a tree structure or if it will be flat.
+	DownloadJobArtefactWithTree(ctx context.Context, jobName string, maintainTreeLocation bool, outputDirectory string, artefactManager *client.ArtefactManagerItem) error
 	// ListJobArtefacts lists all artefact managers associated with a particular job.
 	ListJobArtefacts(ctx context.Context, jobName string) (pagination.IPaginatorAndPageFetcher, error)
-	// DownloadAllJobArtefacts downloads all the artefacts produced for a particular job and puts them in an output directory.
+	// DownloadAllJobArtefacts downloads all the artefacts produced for a particular job and puts them in an output directory as a flat list.
 	DownloadAllJobArtefacts(ctx context.Context, jobName string, outputDirectory string) error
+	// DownloadAllJobArtefactsWithTree downloads all the artefacts produced for a particular job and puts them in an output directory.
+	// maintainTreeStructure specifies whether to keep the tree structure of the artefacts or not in the output directory.
+	DownloadAllJobArtefactsWithTree(ctx context.Context, jobName string, maintainTreeStructure bool, outputDirectory string) error
 }

utils/go.mod

@@ -3,15 +3,15 @@ module github.com/ARM-software/embedded-development-services-client-utils/utils
 go 1.23
 
 require (
-	github.com/ARM-software/embedded-development-services-client/client v1.35.3
-	github.com/ARM-software/golang-utils/utils v1.73.2
+	github.com/ARM-software/embedded-development-services-client/client v1.36.1
+	github.com/ARM-software/golang-utils/utils v1.74.1
 	github.com/go-faker/faker/v4 v4.5.0
 	github.com/go-logr/logr v1.4.2
 	github.com/golang/mock v1.6.0
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/atomic v1.11.0
 	go.uber.org/goleak v1.3.0
-	golang.org/x/sync v0.8.0
+	golang.org/x/sync v0.9.0
 )
 
 require (
@@ -68,10 +68,10 @@ require (
 	github.com/zailic/slogr v0.0.2-alpha // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )