Issue 480, improve performance of update-index (#496)

## Fixes
#480 

## Changes
- maintain version info in index.pidx with current downloaded pdsc files
- added list of pdsc's with download errors to return to old version in
index.pidx

## Checklist
<!-- Put an `x` in the boxes. All tasks must be completed and boxes
checked before merging. -->
- [x] 🤖 This change is covered by unit tests (if applicable).
- [x] 🤹 Manual testing has been performed (if necessary).
- [ ] 🛡️ Security impacts have been considered (if relevant).
- [ ] 📖 Documentation updates are complete (if required).
- [ ] 🧠 Third-party dependencies and TPIP updated (if required).

Author: bgn42

cmd/commands/connection.go

@@ -41,18 +41,7 @@ The url is optional. Ex "cpackget connection https://www.keil.com/pack"`,
 		var err error
 
 		if indexPath == "" { // try to fetch from environment
-			err = configureInstaller(cmd, args)
-			if err != nil {
-				return err
-			}
-			if err := installer.ReadIndexFiles(); err != nil {
-				return err
-			}
-		}
-
-		indexPath, err = installer.GetIndexPath(indexPath)
-		if err != nil {
-			return err
+			indexPath = installer.ConnectionTryURL
 		}
 
 		err = utils.CheckConnection(indexPath, viper.GetInt("timeout"))

cmd/commands/connection_test.go

@@ -39,8 +39,8 @@ var connectionCmdTests = []TestCase{
 		noCleanup: true,
 		setUpFunc: func(t *TestCase) {
 			server := NewServer()
-			t.args = append(t.args, server.URL()+installer.PublicIndex)
-			server.AddRoute(installer.PublicIndex, []byte(`<?xml version="1.0" encoding="UTF-8" ?>
+			t.args = append(t.args, server.URL()+installer.PublicIndexName)
+			server.AddRoute(installer.PublicIndexName, []byte(`<?xml version="1.0" encoding="UTF-8" ?>
 <index schemaVersion="1.1.0" xs:noNamespaceSchemaLocation="PackIndex.xsd" xmlns:xs="https://www.w3.org/2001/XMLSchema-instance">
 <vendor>TheVendor</vendor>
 <url>https://www.keil.com/</url>

cmd/commands/init_test.go

@@ -18,7 +18,7 @@ import (
 
 var (
 	pidxFilePath         = filepath.Join(testingDir, "SamplePublicIndex.pidx")
-	notFoundPidxFilePath = filepath.Join("path", "to", installer.PublicIndex)
+	notFoundPidxFilePath = filepath.Join("path", "to", installer.PublicIndexName)
 )
 
 var initCmdTests = []TestCase{
@@ -33,12 +33,12 @@ var initCmdTests = []TestCase{
 		expectedErr: nil,
 	},
 	{
-		name: "test create using an " + installer.PublicIndex,
+		name: "test create using an " + installer.PublicIndexName,
 		args: []string{"init"},
 		setUpFunc: func(t *TestCase) {
 			server := NewServer()
-			t.args = append(t.args, server.URL()+installer.PublicIndex)
-			server.AddRoute(installer.PublicIndex, []byte(`<?xml version="1.0" encoding="UTF-8" ?>
+			t.args = append(t.args, server.URL()+installer.PublicIndexName)
+			server.AddRoute(installer.PublicIndexName, []byte(`<?xml version="1.0" encoding="UTF-8" ?>
 <index schemaVersion="1.1.0" xs:noNamespaceSchemaLocation="PackIndex.xsd" xmlns:xs="https://www.w3.org/2001/XMLSchema-instance">
 <vendor>TheVendor</vendor>
 <url>https://the.vendor/</url>
@@ -50,12 +50,12 @@ var initCmdTests = []TestCase{
 		},
 	},
 	{
-		name:           "test create using local " + installer.PublicIndex,
+		name:           "test create using local " + installer.PublicIndexName,
 		args:           []string{"init", pidxFilePath},
 		createPackRoot: true,
 	},
 	{
-		name:           "test create using local " + installer.PublicIndex + " that does not exist",
+		name:           "test create using local " + installer.PublicIndexName + " that does not exist",
 		args:           []string{"init", notFoundPidxFilePath},
 		createPackRoot: true,
 		expectedErr:    errs.ErrFileNotFoundUseInit,

cmd/commands/root.go

@@ -71,7 +71,7 @@ func configureInstaller(cmd *cobra.Command, args []string) error {
 
 	if targetPackRoot == installer.GetDefaultCmsisPackRoot() {
 		// If using the default pack root path and the public index is not found, initialize it
-		if !checkConnection && !utils.FileExists(filepath.Join(targetPackRoot, ".Web", installer.PublicIndex)) {
+		if !checkConnection && !utils.FileExists(filepath.Join(targetPackRoot, ".Web", installer.PublicIndexName)) {
 			err := installer.SetPackRoot(targetPackRoot, true)
 			if err != nil {
 				return err

cmd/commands/update_index.go

@@ -50,7 +50,7 @@ var UpdateIndexCmd = &cobra.Command{
 }
 
 func getLongUpdateDescription() string {
-	return `Updates the public index in ` + os.Getenv("CMSIS_PACK_ROOT") + "/.Web/" + installer.PublicIndex + " using the URL in <url> tag inside " + installer.PublicIndex + `.
+	return `Updates the public index in ` + os.Getenv("CMSIS_PACK_ROOT") + "/.Web/" + installer.PublicIndexName + " using the URL in <url> tag inside " + installer.PublicIndexName + `.
 By default it will also check if all PDSC files under .Web/ need update as well. This can be disabled via the "--sparse" flag.`
 }
 

cmd/commands/update_index_test.go

@@ -16,7 +16,7 @@ var updateIndexServer Server
 var updateIndexCmdTests = []TestCase{
 	{
 		name:        "test no parameter is required",
-		args:        []string{"update-index", installer.PublicIndex},
+		args:        []string{"update-index", installer.PublicIndexName},
 		expectedErr: errors.New("accepts 0 arg(s), received 1"),
 	},
 	{
@@ -29,7 +29,7 @@ var updateIndexCmdTests = []TestCase{
 		args:           []string{"update-index"},
 		createPackRoot: true,
 		defaultMode:    true,
-		expectedStdout: []string{"Updating public index", "Downloading " + installer.PublicIndex},
+		expectedStdout: []string{"Updating public index", "Downloading " + installer.PublicIndexName},
 		setUpFunc: func(t *TestCase) {
 			indexContent := `<?xml version="1.0" encoding="UTF-8" ?>
 <index schemaVersion="1.1.0" xs:noNamespaceSchemaLocation="PackIndex.xsd" xmlns:xs="http://www.w3.org/2001/XMLSchema-instance">
@@ -43,14 +43,14 @@ var updateIndexCmdTests = []TestCase{
 			indexContent = fmt.Sprintf(indexContent, updateIndexServer.URL())
 			_ = os.WriteFile(installer.Installation.PublicIndex, []byte(indexContent), 0600)
 
-			updateIndexServer.AddRoute(installer.PublicIndex, []byte(indexContent))
+			updateIndexServer.AddRoute(installer.PublicIndexName, []byte(indexContent))
 		},
 	},
 	{
 		name:           "test updating index",
 		args:           []string{"update-index"},
 		createPackRoot: true,
-		expectedStdout: []string{"Updating public index", "Downloading " + installer.PublicIndex},
+		expectedStdout: []string{"Updating public index", "Downloading " + installer.PublicIndexName},
 		setUpFunc: func(t *TestCase) {
 			indexContent := `<?xml version="1.0" encoding="UTF-8" ?>
 <index schemaVersion="1.1.0" xs:noNamespaceSchemaLocation="PackIndex.xsd" xmlns:xs="http://www.w3.org/2001/XMLSchema-instance">
@@ -64,7 +64,7 @@ var updateIndexCmdTests = []TestCase{
 			indexContent = fmt.Sprintf(indexContent, updateIndexServer.URL())
 			_ = os.WriteFile(installer.Installation.PublicIndex, []byte(indexContent), 0600)
 
-			updateIndexServer.AddRoute(installer.PublicIndex, []byte(indexContent))
+			updateIndexServer.AddRoute(installer.PublicIndexName, []byte(indexContent))
 		},
 	},
 }

cmd/installer/pack.go

@@ -32,9 +32,6 @@ type PackType struct {
 	// IsPublic tells whether the pack exists in the public index or not
 	IsPublic bool
 
-	// isInstalled tells whether the pack is already installed
-	isInstalled bool
-
 	// isDownloaded tells whether the file needed to be downloaded from a server
 	isDownloaded bool
 
@@ -51,6 +48,12 @@ type PackType struct {
 	// targetVersion is the most recent version of a pack in case exactVersion==true
 	targetVersion string
 
+	// isInstalled tells whether the pack is already installed
+	isInstalled bool
+
+	// list of all the packs that are installed
+	installedVersions []string
+
 	// path points to a file in the local system, whether or not it's local
 	path string
 
@@ -73,6 +76,20 @@ type PackType struct {
 	}
 }
 
+func isGlobal(packPath string) (bool, error) {
+	info, err := utils.ExtractPackInfo(packPath)
+	if err != nil {
+		return false, err
+	}
+	if info.IsPackID {
+		return true, nil
+	}
+	if !strings.HasPrefix(info.Location, "http://") && !strings.HasPrefix(info.Location, "https://") && strings.HasPrefix(info.Location, "file://") {
+		return true, nil
+	}
+	return false, nil
+}
+
 // preparePack prepares a PackType object based on the provided parameters,
 // it does some sanity validation regarding pack name
 // and check if it's public and if it's installed or not
@@ -119,8 +136,8 @@ func preparePack(packPath string, toBeRemoved, forceLatest, noLocal, nometa bool
 		info.VersionModifier = utils.LatestVersion
 	}
 	pack.URL = info.Location
-	pack.Name = info.Pack
 	pack.Vendor = info.Vendor
+	pack.Name = info.Pack
 	pack.Version = info.Version
 	pack.versionModifier = info.VersionModifier
 	pack.isPackID = info.IsPackID
@@ -129,7 +146,8 @@ func preparePack(packPath string, toBeRemoved, forceLatest, noLocal, nometa bool
 		pack.IsLocallySourced = true
 	}
 
-	if pack.IsPublic, err = Installation.packIsPublic(pack, timeout); err != nil {
+	var pdscTag xml.PdscTag
+	if pack.IsPublic, err = Installation.packIsPublic(pack, &pdscTag); err != nil {
 		return pack, err
 	}
 
@@ -139,7 +157,23 @@ func preparePack(packPath string, toBeRemoved, forceLatest, noLocal, nometa bool
 		}
 	}
 
-	pack.isInstalled = Installation.PackIsInstalled(pack, noLocal)
+	if pdscTag.URL != "" {
+		err = Installation.downloadPdscFile(pdscTag, false, timeout)
+		if err != nil {
+			return pack, err
+		}
+	}
+
+	pack.isInstalled, pack.installedVersions = Installation.PackIsInstalled(pack, noLocal)
+
+	if pdscTag.Vendor != "" {
+		pack.Vendor = pdscTag.Vendor
+	}
+	if pdscTag.Name != "" {
+		pack.Name = pdscTag.Name
+	}
+	// pack.Version = pdscTag.Version
+	// pack.URL = pdscTag.URL
 
 	return pack, nil
 }
@@ -343,8 +377,11 @@ func (p *PackType) install(installation *PacksInstallationType, checkEula bool)
 		return err
 	}
 
-	log.Debugf("Extracting files from \"%s\" to \"%s\"", p.path, packHomeDir)
-	log.Infof("Extracting files to %s...", packHomeDir)
+	if log.IsLevelEnabled(log.DebugLevel) {
+		log.Debugf("Extracting files from \"%s\" to \"%s\"", p.path, packHomeDir)
+	} else {
+		log.Infof("Extracting files to %s...", packHomeDir)
+	}
 	// Avoid repeated calls to IsTerminalInteractive
 	// as it cleans the stdout buffer
 	interactiveTerminal := utils.IsTerminalInteractive()
@@ -650,7 +687,7 @@ func (p *PackType) loadDependencies(nometa bool) error {
 		} else {
 			pack.versionModifier = utils.LatestVersion
 		}
-		if Installation.PackIsInstalled(pack, false) {
+		if ok, _ := Installation.PackIsInstalled(pack, false); ok {
 			p.Requirements.packages = append(p.Requirements.packages, struct {
 				info      []string
 				installed bool