diff --git a/DESIGN.md b/DESIGN.md index 8d63293f..e1efc8c5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -203,7 +203,9 @@ The `configureInstaller` pre-run hook: | --- | --- | | `list` (default) | Lists all installed packs | | `list --cached` | Lists packs in `.Download/` | -| `list --public` | Lists all packs from the public index | +| `list --public` | Lists all non-deprecated packs from the public index | +| `list --public --deprecated` | Lists deprecated packs from the public index | +| `list --deprecated` | Lists deprecated packs from the public index | | `list --updates` | Lists packs with newer versions available | | `list --filter` | Filters results (case-sensitive, accepts multiple expressions) | | `list-required` | Lists dependencies of installed packs | @@ -239,11 +241,15 @@ type PacksInstallationType struct { - `UpdatePack()` — Updates one or all packs to latest versions - `InitializeCache()` — Builds `cache.pidx` from existing PDSC files in `.Web/` - `CheckConcurrency()` — Validates and adjusts the concurrent-downloads setting -- `DownloadPDSCFiles()` — Downloads all PDSC files from the public index in parallel +- `DownloadPDSCFiles()` — Downloads all PDSC files from the public + index in parallel, optionally skipping deprecated packs - `UpdateInstalledPDSCFiles()` — Refreshes already-cached PDSC files from the index - `UpdatePublicIndexIfOnline()` — Updates the public index only when connectivity is available -- `UpdatePublicIndex()` — Downloads and updates the public index and PDSC files -- `ListInstalledPacks()` — Lists packs with various filter modes +- `UpdatePublicIndex()` — Downloads and updates the public index and + PDSC files, with option to skip deprecated PDSC files +- `ListInstalledPacks()` — Lists packs with various filter modes; + supports `--deprecated` flag to show only deprecated packs + (hidden by default in `--public` listing) - `FindPackURL()` — Resolves a pack ID to a download URL from the index - `SetPackRoot()` — Initializes the `Installation` singleton and directory paths - `ReadIndexFiles()` — Loads `index.pidx`, `local_repository.pidx`, and `cache.pidx` @@ -329,17 +335,19 @@ type PidxXML struct { Pdscs []PdscTag // List of all pack references } // Internal lookup maps for O(1) access - pdscList map[string][]int // key → indices - pdscListName map[string][]int // vendor.name → indices + pdscList map[string][]PdscTag // key → PdscTags + pdscListName map[string]string // vendor.name → key + deprecatedDate time.Time // today UTC, set once per Read() } type PdscTag struct { - URL string `xml:"url,attr"` - Vendor string `xml:"vendor,attr"` - Name string `xml:"name,attr"` - Version string `xml:"version,attr"` - Deprecated string `xml:"deprecated,attr,omitempty"` - Replacement string `xml:"replacement,attr,omitempty"` + URL string `xml:"url,attr"` + Vendor string `xml:"vendor,attr"` + Name string `xml:"name,attr"` + Version string `xml:"version,attr"` + Deprecated string `xml:"deprecated,attr,omitempty"` + Replacement string `xml:"replacement,attr,omitempty"` + isDeprecated bool // cached flag, computed on insert } ``` @@ -359,6 +367,11 @@ type PdscTag struct { - `YamlPackID()` — Returns `Vendor::Name@Version` format - `PackURL()` — Constructs the full `.pack` download URL (PdscTag method) - `PdscFileName()` — Returns the `.pdsc` filename (PdscTag method) +- `IsDeprecated()` — Returns the cached deprecated flag. + Computed via `computeIsDeprecated()` when a PdscTag is + inserted (`Read`, `AddPdsc`, `AddReplacePdsc`). + Uses `PidxXML.deprecatedDate` (today UTC, set once per + `NewPidxXML`/`Read`) as reference (PdscTag method) ### 7.2 PDSC — Pack Description (`pdsc.go`) @@ -666,7 +679,8 @@ All errors are predefined constants in `errors.go`, allowing consistent error ch Helper functions: - `Is()` — Wraps `errors.Is()` for convenience -- `AlreadyLogged()` — Wraps errors to prevent the same message from being logged twice as the error travels up the call stack +- `AlreadyLogged()` — Wraps errors to prevent the same message from + being logged twice as the error travels up the call stack --- @@ -706,6 +720,7 @@ installer.UpdatePublicIndex() ├── Download new index.pidx from upstream URL ├── Compare old vs. new entries ├── Download updated/new PDSC files (concurrent, via semaphore) + │ └── Skip PDSC files where Deprecated date ≤ today ├── Update cache.pidx to reflect changes └── Remove deprecated entries ``` diff --git a/cmd/commands/init.go b/cmd/commands/init.go index 3c0fdb7d..a50ea742 100644 --- a/cmd/commands/init.go +++ b/cmd/commands/init.go @@ -56,7 +56,7 @@ The index-url is mandatory. Ex "cpackget init --pack-root path/to/mypackroot htt return err } - err = installer.UpdatePublicIndex(indexPath, true, initCmdFlags.downloadPdscFiles, false, true, true, initCmdFlags.insecureSkipVerify, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) + err = installer.UpdatePublicIndex(indexPath, true, initCmdFlags.downloadPdscFiles, false, true, true, true, initCmdFlags.insecureSkipVerify, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) return err }, } diff --git a/cmd/commands/list.go b/cmd/commands/list.go index f8365d79..042ab1ad 100644 --- a/cmd/commands/list.go +++ b/cmd/commands/list.go @@ -19,18 +19,24 @@ var listCmdFlags struct { // listCached tells whether listing all cached packs listCached bool + // listDeprecated tells whether listing all deprecated packs + listDeprecated bool + // listFilter is a set of words by which to filter listed packs listFilter string } var ListCmd = &cobra.Command{ - Use: "list [--cached|--public|--updates]", + Use: "list [--cached|--public|--updates|--deprecated] [--filter ]", Short: "List installed packs", Long: "List all installed packs and optionally cached packs or those for which updates are available", Args: cobra.MaximumNArgs(0), PersistentPreRunE: configureInstaller, RunE: func(cmd *cobra.Command, args []string) error { - return installer.ListInstalledPacks(listCmdFlags.listCached, listCmdFlags.listPublic, listCmdFlags.listUpdates, false, false, listCmdFlags.listFilter) + if listCmdFlags.listDeprecated { + listCmdFlags.listPublic = true + } + return installer.ListInstalledPacks(listCmdFlags.listCached, listCmdFlags.listPublic, listCmdFlags.listUpdates, listCmdFlags.listDeprecated, false, false, listCmdFlags.listFilter) }, } @@ -41,7 +47,7 @@ var listRequiredCmd = &cobra.Command{ Args: cobra.MaximumNArgs(0), PersistentPreRunE: configureInstaller, RunE: func(cmd *cobra.Command, args []string) error { - return installer.ListInstalledPacks(listCmdFlags.listCached, listCmdFlags.listPublic, listCmdFlags.listUpdates, true, false, listCmdFlags.listFilter) + return installer.ListInstalledPacks(listCmdFlags.listCached, listCmdFlags.listPublic, listCmdFlags.listUpdates, listCmdFlags.listDeprecated, true, false, listCmdFlags.listFilter) }, } @@ -49,6 +55,7 @@ func init() { ListCmd.Flags().BoolVarP(&listCmdFlags.listCached, "cached", "c", false, "list only cached packs") ListCmd.Flags().BoolVarP(&listCmdFlags.listPublic, "public", "p", false, "list packs in the public index") ListCmd.Flags().BoolVarP(&listCmdFlags.listUpdates, "updates", "u", false, "list packs which have newer versions") + ListCmd.Flags().BoolVarP(&listCmdFlags.listDeprecated, "deprecated", "d", false, "list only deprecated packs") ListCmd.Flags().StringVarP(&listCmdFlags.listFilter, "filter", "f", "", "filter results (case sensitive, accepts several expressions)") ListCmd.AddCommand(listRequiredCmd) diff --git a/cmd/commands/root.go b/cmd/commands/root.go index d468eab0..89ed6ae4 100644 --- a/cmd/commands/root.go +++ b/cmd/commands/root.go @@ -79,7 +79,7 @@ func configureInstaller(cmd *cobra.Command, args []string) error { // Exclude index updating commands to not double update if cmd.Name() != "init" && cmd.Name() != "index" && cmd.Name() != "update-index" && cmd.Name() != "list" { installer.UnlockPackRoot() - err = installer.UpdatePublicIndex(installer.ActualPublicIndex, true, false, false, false, true, false, 0, 0) + err = installer.UpdatePublicIndex(installer.ActualPublicIndex, true, false, false, true, false, true, false, 0, 0) if err != nil { return err } diff --git a/cmd/commands/update_index.go b/cmd/commands/update_index.go index 1e85832a..84976428 100644 --- a/cmd/commands/update_index.go +++ b/cmd/commands/update_index.go @@ -47,7 +47,7 @@ var UpdateIndexCmd = &cobra.Command{ return err } - err = installer.UpdatePublicIndex("", updateIndexCmdFlags.sparse, false, updateIndexCmdFlags.downloadUpdatePdscFiles, true, true, updateIndexCmdFlags.insecureSkipVerify, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) + err = installer.UpdatePublicIndex("", updateIndexCmdFlags.sparse, false, updateIndexCmdFlags.downloadUpdatePdscFiles, updateIndexCmdFlags.downloadUpdatePdscFiles, true, true, updateIndexCmdFlags.insecureSkipVerify, viper.GetInt("concurrent-downloads"), viper.GetInt("timeout")) return err }, } diff --git a/cmd/installer/root.go b/cmd/installer/root.go index 9f9edb41..7b5daf39 100644 --- a/cmd/installer/root.go +++ b/cmd/installer/root.go @@ -674,19 +674,34 @@ func CheckConcurrency(concurrency int) int { // // Parameters: // - skipInstalledPdscFiles: If true, skips downloading PDSC files that are already installed. +// - skipDeprecatedPdscFiles: If true, skips downloading PDSC files that are marked as deprecated. // - insecureSkipVerify: If true, skips TLS certificate verification for HTTPS downloads. // - concurrency: The number of concurrent downloads to allow. If 0, downloads are sequential. // - timeout: The timeout for each download operation. // // Returns: // - An error if there is an issue reading the public index XML or acquiring the semaphore. -func DownloadPDSCFiles(skipInstalledPdscFiles, insecureSkipVerify bool, concurrency int, timeout int) error { +func DownloadPDSCFiles(skipInstalledPdscFiles, skipDeprecatedPdscFiles, insecureSkipVerify bool, concurrency int, timeout int) error { log.Info("Downloading all PDSC files available on the public index") // if err := Installation.PublicIndexXML.Read(); err != nil { // return err // } - pdscTags := Installation.PublicIndexXML.ListPdscTags() + allPdscTags := Installation.PublicIndexXML.ListPdscTags() + + // Filter out deprecated tags upfront if requested, so the job count + // used for progress reporting matches the actual number of downloads. + var pdscTags []xml.PdscTag + if skipDeprecatedPdscFiles { + for _, t := range allPdscTags { + if !t.IsDeprecated() { + pdscTags = append(pdscTags, t) + } + } + } else { + pdscTags = allPdscTags + } + numPdsc := len(pdscTags) if numPdsc == 0 { log.Info("(no packs in public index)") @@ -940,7 +955,7 @@ func UpdatePublicIndexIfOnline() error { err = Installation.checkUpdateCfg(&updateConf, true) if err != nil { UnlockPackRoot() - err1 := UpdatePublicIndex(ActualPublicIndex, false, false, false, false, false, false, 0, 0) + err1 := UpdatePublicIndex(ActualPublicIndex, false, false, false, true, false, false, false, 0, 0) if err1 != nil { log.Warnf("Cannot update public index: %v", err1) return nil @@ -954,7 +969,7 @@ func UpdatePublicIndexIfOnline() error { // if public index does not or not yet exist then download without check if !utils.FileExists(Installation.PublicIndex) { UnlockPackRoot() - err1 := UpdatePublicIndex(ActualPublicIndex, false, false, false, false, false, false, 0, 0) + err1 := UpdatePublicIndex(ActualPublicIndex, false, false, false, true, false, false, false, 0, 0) if err1 != nil { log.Warnf("Cannot update public index: %v", err1) return nil @@ -974,6 +989,7 @@ func UpdatePublicIndexIfOnline() error { // - sparse: A boolean flag to indicate whether to perform a sparse update. // - downloadPdsc: A boolean flag to indicate whether to download PDSC files. // - downloadRemainingPdscFiles: A boolean flag to indicate whether to download all remaining PDSC files. +// - skipDeprecatedPdscFiles: If true, skips downloading deprecated PDSC files. // - updatePrivatePdsc: If true, updates private PDSC files during the update process. // - showInfo: If true, logs informational messages about the download. // - insecureSkipVerify: A boolean flag to indicate whether to skip TLS certificate verification for HTTPS downloads. @@ -982,7 +998,7 @@ func UpdatePublicIndexIfOnline() error { // // Returns: // - error: An error if the update fails, otherwise nil. -func UpdatePublicIndex(indexPath string, sparse, downloadPdsc, downloadRemainingPdscFiles, updatePrivatePdsc, showInfo, insecureSkipVerify bool, concurrency int, timeout int) error { +func UpdatePublicIndex(indexPath string, sparse, downloadPdsc, downloadRemainingPdscFiles, skipDeprecatedPdscFiles, updatePrivatePdsc, showInfo, insecureSkipVerify bool, concurrency int, timeout int) error { // For backwards compatibility, allow indexPath to be a file, but ideally it should be empty if indexPath == "" { indexPath = strings.TrimSuffix(Installation.PublicIndexXML.URL, "/") + "/" + PublicIndexName @@ -1043,7 +1059,7 @@ func UpdatePublicIndex(indexPath string, sparse, downloadPdsc, downloadRemaining utils.SetReadOnly(Installation.PublicIndex) if downloadPdsc { - err = DownloadPDSCFiles(false, insecureSkipVerify, concurrency, timeout) + err = DownloadPDSCFiles(false, skipDeprecatedPdscFiles, insecureSkipVerify, concurrency, timeout) if err != nil { return err } @@ -1066,7 +1082,7 @@ func UpdatePublicIndex(indexPath string, sparse, downloadPdsc, downloadRemaining Installation.PublicIndexXML.SetFileName(savedIndexPath) if downloadRemainingPdscFiles { - err = DownloadPDSCFiles(true, insecureSkipVerify, concurrency, timeout) + err = DownloadPDSCFiles(true, skipDeprecatedPdscFiles, insecureSkipVerify, concurrency, timeout) if err != nil { return err } @@ -1190,12 +1206,14 @@ func findInstalledPacks(addLocalPacks, removeDuplicates bool) ([]installedPack, // - listCached: If true, lists the cached packs. // - listPublic: If true, lists the packs from the public index. // - listUpdates: If true, lists the installed packs with available updates. +// - listDeprecated: If true, lists the deprecated packs. // - listRequirements: If true, lists the installed packs with dependencies. +// - testing: If true, skips reading index files (used for testing). // - listFilter: A string to filter the packs by. // // Returns: // - error: An error if any occurs during the listing process. -func ListInstalledPacks(listCached, listPublic, listUpdates, listRequirements, testing bool, listFilter string) error { +func ListInstalledPacks(listCached, listPublic, listUpdates, listDeprecated, listRequirements, testing bool, listFilter string) error { log.Debugf("Listing packs") if !testing { @@ -1222,7 +1240,20 @@ func ListInstalledPacks(listCached, listPublic, listUpdates, listRequirements, t }) // List all available packs from the index for _, pdscTag := range pdscTags { + isDeprecated := pdscTag.IsDeprecated() + // Filter by deprecated status: + // --deprecated: show only deprecated packs + // without --deprecated: hide deprecated packs + if listDeprecated && !isDeprecated { + continue + } + if !listDeprecated && isDeprecated { + continue + } logMessage := pdscTag.YamlPackID() + if isDeprecated { + logMessage += " (deprecated)" + } packFilePath := filepath.Join(Installation.DownloadDir, pdscTag.Key()) + utils.PackExtension if ok, _ := Installation.PackIsInstalled(&PackType{PdscTag: pdscTag}, false); ok { diff --git a/cmd/installer/root_pack_list_test.go b/cmd/installer/root_pack_list_test.go index 4e1194c7..c6456290 100644 --- a/cmd/installer/root_pack_list_test.go +++ b/cmd/installer/root_pack_list_test.go @@ -24,6 +24,7 @@ var ( ListFilter = "" ListPublic = true ListUpdates = true + ListDeprecated = true ListRequirements = true ) @@ -38,7 +39,7 @@ func ExampleListInstalledPacks() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter) + _ = installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter) // Output: // I: Listing installed packs // I: (no packs installed) @@ -54,7 +55,7 @@ func ExampleListInstalledPacks_emptyCache() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter) + _ = installer.ListInstalledPacks(ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter) // Output: // I: Listing cached packs // I: (no packs cached) @@ -70,7 +71,7 @@ func ExampleListInstalledPacks_emptyPublicIndex() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListRequirements, true, ListFilter) + _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter) // Output: // I: Listing packs from the public index // I: (no packs in public index) @@ -120,7 +121,7 @@ func ExampleListInstalledPacks_list() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListRequirements, true, ListFilter) + _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter) // Output: // I: Listing packs from the public index // I: TheVendor::PublicLocalPack@1.2.3 (cached) @@ -158,7 +159,7 @@ func ExampleListInstalledPacks_listCached() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter) + _ = installer.ListInstalledPacks(ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter) // Output: // I: Listing cached packs // I: TheVendor::PublicLocalPack@1.2.3 @@ -207,7 +208,7 @@ func TestListInstalledPacks(t *testing.T) { var buf bytes.Buffer log.SetOutput(&buf) defer log.SetOutput(io.Discard) - assert.Nil(installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter)) + assert.Nil(installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter)) stdout := buf.String() assert.Contains(stdout, "I: Listing installed packs") assert.Contains(stdout, fmt.Sprintf("I: TheVendor::PackName@1.2.3 (installed via %s)", expectedPdscAbsPath)) @@ -238,7 +239,7 @@ func TestListInstalledPacks(t *testing.T) { var buf bytes.Buffer log.SetOutput(&buf) defer log.SetOutput(io.Discard) - assert.Nil(installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter)) + assert.Nil(installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter)) stdout := buf.String() assert.Contains(stdout, "I: Listing installed packs") assert.Contains(stdout, fmt.Sprintf("I: TheVendor::PackName@1.2.3 (installed via %s)", expectedPdscAbsPath)) @@ -248,11 +249,99 @@ func TestListInstalledPacks(t *testing.T) { assert.Nil(pdscXML.Read()) pdscXML.ReleasesTag.Releases[0].Version = "1.2.4" assert.Nil(utils.WriteXML(pdscPath, pdscXML)) - assert.Nil(installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter)) + assert.Nil(installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter)) stdout = buf.String() assert.Contains(stdout, "I: Listing installed packs") assert.Contains(stdout, fmt.Sprintf("I: TheVendor::PackName@1.2.4 (installed via %s)", expectedPdscAbsPath)) }) + + t.Run("test list public deprecated only shows deprecated packs", func(t *testing.T) { + localTestingDir := "test-list-deprecated-packs" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + assert.Nil(installer.ReadIndexFiles()) + defer removePackRoot(localTestingDir) + + // Add a deprecated pack and a non-deprecated pack to the public index + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "DeprecatedPack", + Version: "1.0.0", + Deprecated: "2020-01-01", + })) + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "ActivePack", + Version: "2.0.0", + })) + assert.Nil(installer.Installation.PublicIndexXML.Write()) + + var buf bytes.Buffer + log.SetOutput(&buf) + defer log.SetOutput(io.Discard) + + // list --public --deprecated should only show deprecated packs + assert.Nil(installer.ListInstalledPacks(!ListCached, ListPublic, !ListUpdates, ListDeprecated, !ListRequirements, true, ListFilter)) + stdout := buf.String() + assert.Contains(stdout, "TheVendor::DeprecatedPack@1.0.0 (deprecated)") + assert.NotContains(stdout, "TheVendor::ActivePack@2.0.0") + }) + + t.Run("test list public without deprecated hides deprecated packs", func(t *testing.T) { + localTestingDir := "test-list-public-no-deprecated-flag" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + assert.Nil(installer.ReadIndexFiles()) + defer removePackRoot(localTestingDir) + + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "DeprecatedPack", + Version: "1.0.0", + Deprecated: "2020-01-01", + })) + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "ActivePack", + Version: "2.0.0", + })) + assert.Nil(installer.Installation.PublicIndexXML.Write()) + + var buf bytes.Buffer + log.SetOutput(&buf) + defer log.SetOutput(io.Discard) + + // list --public (without --deprecated) should hide deprecated packs + assert.Nil(installer.ListInstalledPacks(!ListCached, ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter)) + stdout := buf.String() + assert.NotContains(stdout, "TheVendor::DeprecatedPack@1.0.0") + assert.Contains(stdout, "TheVendor::ActivePack@2.0.0") + }) + + t.Run("test list public deprecated with future date is not deprecated", func(t *testing.T) { + localTestingDir := "test-list-deprecated-future-date" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + assert.Nil(installer.ReadIndexFiles()) + defer removePackRoot(localTestingDir) + + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "FutureDeprecatedPack", + Version: "1.0.0", + Deprecated: "2099-12-31", + })) + assert.Nil(installer.Installation.PublicIndexXML.Write()) + + var buf bytes.Buffer + log.SetOutput(&buf) + defer log.SetOutput(io.Discard) + + // list --public --deprecated should NOT show packs with future deprecation date + assert.Nil(installer.ListInstalledPacks(!ListCached, ListPublic, !ListUpdates, ListDeprecated, !ListRequirements, true, ListFilter)) + stdout := buf.String() + assert.NotContains(stdout, "TheVendor::FutureDeprecatedPack@1.0.0") + }) } func ExampleListInstalledPacks_listMalformedInstalledPacks() { @@ -287,7 +376,7 @@ func ExampleListInstalledPacks_listMalformedInstalledPacks() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, ListFilter) + _ = installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, ListFilter) // Output: // I: Listing installed packs // E: _TheVendor::_PublicLocalPack@1.2.3.4 - error: pack version incorrect format @@ -334,7 +423,7 @@ func ExampleListInstalledPacks_filter() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListRequirements, true, "1.2.4") + _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, "1.2.4") // Output: // I: Listing packs from the public index, filtering by "1.2.4" // I: TheVendor::PublicLocalPack@1.2.4 (installed) @@ -372,7 +461,7 @@ func ExampleListInstalledPacks_filterErrorPackages() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, "TheVendor") + _ = installer.ListInstalledPacks(!ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, "TheVendor") // Output: // I: Listing installed packs, filtering by "TheVendor" // E: _TheVendor::_PublicLocalPack@1.2.3.4 - error: pack version incorrect format @@ -418,7 +507,7 @@ func ExampleListInstalledPacks_filterInvalidChars() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListRequirements, true, "@ :") + _ = installer.ListInstalledPacks(ListCached, ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, "@ :") // Output: // I: Listing packs from the public index, filtering by "@ :" } @@ -453,7 +542,7 @@ func ExampleListInstalledPacks_filteradditionalMessages() { log.SetOutput(os.Stdout) defer log.SetOutput(io.Discard) - _ = installer.ListInstalledPacks(ListCached, !ListPublic, !ListUpdates, !ListRequirements, true, "(installed)") + _ = installer.ListInstalledPacks(ListCached, !ListPublic, !ListUpdates, !ListDeprecated, !ListRequirements, true, "(installed)") // Output: // I: Listing cached packs, filtering by "(installed)" } diff --git a/cmd/installer/root_test.go b/cmd/installer/root_test.go index 2a5e7d46..cf751134 100644 --- a/cmd/installer/root_test.go +++ b/cmd/installer/root_test.go @@ -738,6 +738,7 @@ func TestUpdatePublicIndex(t *testing.T) { var Sparse = true var DownloadPdsc = false var DownloadRemainingPdscFiles = true + var skipDeprecatedPdscFiles = true var UpdatePrivatePdsc = true var ShowInfo = true var Concurrency = 0 @@ -776,7 +777,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexPath := server.URL() + "this-file-does-not-exist" - err := installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err := installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.NotNil(err) assert.Equal(errors.Unwrap(err), errs.ErrBadRequest) @@ -796,7 +797,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(installer.PublicIndexName, indexContent) indexPath := indexServer.URL() + installer.PublicIndexName - err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.NotNil(err) assert.Contains(err.Error(), "XML syntax error on line 3: unexpected EOF") @@ -815,7 +816,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(installer.PublicIndexName, indexContent) indexPath := indexServer.URL() + installer.PublicIndexName - err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.Nil(err) @@ -846,12 +847,80 @@ func TestUpdatePublicIndex(t *testing.T) { assert.Nil(err) indexServer.AddRoute("TheVendor.PublicLocalPack.pdsc", pdscContent) - err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.Nil(err) assert.True(utils.FileExists(path.Join(localTestingDir, ".Web", "TheVendor.PublicLocalPack.pdsc"))) }) + t.Run("test all-pdsc-files skips deprecated packs", func(t *testing.T) { + localTestingDir := "test-all-pdsc-files-skips-deprecated" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + assert.Nil(installer.ReadIndexFiles()) + defer os.RemoveAll(localTestingDir) + + indexServer := NewServer() + + pdscContent, err := os.ReadFile(publicLocalPack123Pdsc) + assert.Nil(err) + indexServer.AddRoute("TheVendor.ActivePack.pdsc", pdscContent) + indexServer.AddRoute("TheVendor.DeprecatedPack.pdsc", pdscContent) + + // Add both a deprecated and a non-deprecated pack to the public index + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "ActivePack", + Version: "1.2.3", + URL: indexServer.URL(), + })) + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "DeprecatedPack", + Version: "1.0.0", + Deprecated: "2020-01-01", + URL: indexServer.URL(), + })) + + // DownloadPDSCFiles with skipDeprecated=true should skip the deprecated pack + err = installer.DownloadPDSCFiles(false, skipDeprecatedPdscFiles, !InsecureSkipVerify, Concurrency, Timeout) + assert.Nil(err) + + // Non-deprecated pack PDSC should be downloaded + assert.True(utils.FileExists(filepath.Join(localTestingDir, ".Web", "TheVendor.ActivePack.pdsc"))) + // Deprecated pack PDSC should NOT be downloaded + assert.False(utils.FileExists(filepath.Join(localTestingDir, ".Web", "TheVendor.DeprecatedPack.pdsc"))) + }) + + t.Run("test all-pdsc-files downloads non-deprecated packs with future deprecation date", func(t *testing.T) { + localTestingDir := "test-all-pdsc-files-downloads-future-deprecated" + assert.Nil(installer.SetPackRoot(localTestingDir, CreatePackRoot)) + installer.UnlockPackRoot() + assert.Nil(installer.ReadIndexFiles()) + defer os.RemoveAll(localTestingDir) + + indexServer := NewServer() + + pdscContent, err := os.ReadFile(publicLocalPack123Pdsc) + assert.Nil(err) + indexServer.AddRoute("TheVendor.FutureDeprecatedPack.pdsc", pdscContent) + + // Pack with future deprecation date should not be considered deprecated yet + assert.Nil(installer.Installation.PublicIndexXML.AddPdsc(xml.PdscTag{ + Vendor: "TheVendor", + Name: "FutureDeprecatedPack", + Version: "1.2.3", + Deprecated: "2099-12-31", + URL: indexServer.URL(), + })) + + err = installer.DownloadPDSCFiles(false, skipDeprecatedPdscFiles, !InsecureSkipVerify, Concurrency, Timeout) + assert.Nil(err) + + // Future-deprecated pack should still be downloaded + assert.True(utils.FileExists(filepath.Join(localTestingDir, ".Web", "TheVendor.FutureDeprecatedPack.pdsc"))) + }) + // TODO: this test currently fails because the pdsc file is not found in the public index t.Run("test update-index delete pdsc when not in "+installer.PublicIndexName, func(t *testing.T) { localTestingDir := "test-update-index-delete-pdsc-when-not-in-index" @@ -866,7 +935,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(installer.PublicIndexName, indexContent) indexPath := indexServer.URL() + installer.PublicIndexName - err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.Nil(err) publicIndex := installer.Installation.PublicIndex @@ -880,7 +949,7 @@ func TestUpdatePublicIndex(t *testing.T) { err = utils.CopyFile(pdscPackNotInIndex, filepath.Join(localTestingDir, ".Web", "TheVendor.PackNotInIndex.pdsc")) assert.Nil(err) - err = installer.UpdatePublicIndex(indexPath, false, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, false, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.Nil(err) // assert.False(utils.FileExists(filepath.Join(localTestingDir, ".Web", "TheVendor.PackNotInIndex.pdsc"))) @@ -896,7 +965,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexContent, err := os.ReadFile(samplePublicIndex) assert.Nil(err) - assert.Nil(installer.UpdatePublicIndex(samplePublicIndex, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout)) + assert.Nil(installer.UpdatePublicIndex(samplePublicIndex, Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout)) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -931,7 +1000,7 @@ func TestUpdatePublicIndex(t *testing.T) { assert.Nil(err) indexServer.AddRoute("TheVendor.PublicLocalPack.pdsc", pdscContent) - err = installer.UpdatePublicIndex(indexPath, Sparse, !DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) + err = installer.UpdatePublicIndex(indexPath, Sparse, !DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout) assert.Nil(err) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -965,7 +1034,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(publicConcurrentLocalPdscBase+fmt.Sprint(i)+".pdsc", pdscContent) } - err = installer.UpdatePublicIndex(indexPath, Sparse, !DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, 5, Timeout) + err = installer.UpdatePublicIndex(indexPath, Sparse, !DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, 5, Timeout) assert.Nil(err) assert.True(utils.FileExists(installer.Installation.PublicIndex)) @@ -1040,7 +1109,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(filepath.Base(publicLocalPack124Pdsc), pdscContent) - assert.Nil(installer.UpdatePublicIndex("", !Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout)) + assert.Nil(installer.UpdatePublicIndex("", !Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, Concurrency, Timeout)) // Make sure index.pidx exists and it is updated assert.FileExists(installer.Installation.PublicIndex) @@ -1113,7 +1182,7 @@ func TestUpdatePublicIndex(t *testing.T) { indexServer.AddRoute(pdsc, pdscContent) } - assert.Nil(installer.UpdatePublicIndex("", !Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, 5, Timeout)) + assert.Nil(installer.UpdatePublicIndex("", !Sparse, DownloadPdsc, !DownloadRemainingPdscFiles, skipDeprecatedPdscFiles, !UpdatePrivatePdsc, ShowInfo, !InsecureSkipVerify, 5, Timeout)) // Make sure index.pidx exists and it is updated assert.FileExists(installer.Installation.PublicIndex) diff --git a/cmd/xml/pidx.go b/cmd/xml/pidx.go index 4014bcfe..2ca929a9 100644 --- a/cmd/xml/pidx.go +++ b/cmd/xml/pidx.go @@ -33,10 +33,11 @@ type PidxXML struct { Pdscs []PdscTag `xml:"pdsc"` } `xml:"pindex"` - pdscList map[string][]PdscTag // map of PdscTag.Key() to PdscTag - pdscListName map[string]string // map of lowercase Vendor.Pack to PdscTag.Key() - fileName string - isCache bool // to know if this really is a cache index instead of a PDSC index + pdscList map[string][]PdscTag // map of PdscTag.Key() to PdscTag + pdscListName map[string]string // map of lowercase Vendor.Pack to PdscTag.Key() + fileName string + isCache bool // to know if this really is a cache index instead of a PDSC index + deprecatedDate time.Time // reference date (today UTC) for deprecated evaluation, refreshed once per Read() } // PdscTag maps a tag that goes in PIDX files. @@ -48,6 +49,8 @@ type PdscTag struct { Version string `xml:"version,attr"` Deprecated string `xml:"deprecated,attr,omitempty"` Replacement string `xml:"replacement,attr,omitempty"` + + isDeprecated bool // to mark a tag as deprecated to speed up filtering } // NewPidxXML initializes a new PidxXML object with the given file name. @@ -63,6 +66,7 @@ func NewPidxXML(fileName string, isCache bool) *PidxXML { p := new(PidxXML) p.fileName = fileName p.isCache = isCache + p.deprecatedDate = time.Now().UTC().Truncate(24 * time.Hour) return p } @@ -104,6 +108,7 @@ func (p *PidxXML) Clear() { func (p *PidxXML) AddPdsc(pdsc PdscTag) error { log.Debugf("Adding pdsc tag %v to %q", pdsc, p.fileName) pdsc.Version = utils.SemverStripMeta(pdsc.Version) + pdsc.computeIsDeprecated(p.deprecatedDate) if p.HasPdsc(pdsc) != PdscIndexNotFound { return errs.ErrPdscEntryExists } @@ -136,6 +141,7 @@ func (p *PidxXML) AddPdsc(pdsc PdscTag) error { func (p *PidxXML) AddReplacePdsc(cTag PdscTag) error { log.Debugf("AddReplacePdsc pdsc tag %v to %q", cTag, p.fileName) cTag.Version = utils.SemverStripMeta(cTag.Version) + cTag.computeIsDeprecated(p.deprecatedDate) name := strings.ToLower(cTag.VName()) key, ok := p.pdscListName[name] if ok { @@ -402,6 +408,7 @@ func (p *PidxXML) Read() error { p.pdscList = make(map[string][]PdscTag) p.pdscListName = make(map[string]string) + p.deprecatedDate = time.Now().UTC().Truncate(24 * time.Hour) // Create a new empty Pidx file if it does not exist if !utils.FileExists(p.fileName) { @@ -439,6 +446,7 @@ func (p *PidxXML) Read() error { for _, pdsc := range p.Pindex.Pdscs { pdsc.Version = utils.SemverStripMeta(pdsc.Version) + pdsc.computeIsDeprecated(p.deprecatedDate) key := pdsc.Key() name := strings.ToLower(pdsc.VName()) // log.Debugf("Registring %q", key) @@ -502,3 +510,25 @@ func (p *PdscTag) PackURL() string { func (p *PdscTag) PdscFileName() string { return p.VName() + utils.PdscExtension } + +// computeIsDeprecated evaluates the Deprecated date string against the given +// reference date and caches the result in the isDeprecated field. +func (p *PdscTag) computeIsDeprecated(refDate time.Time) { + if p.Deprecated == "" { + p.isDeprecated = false + return + } + t, err := time.Parse("2006-01-02", p.Deprecated) + if err != nil { + p.isDeprecated = false + return + } + p.isDeprecated = !refDate.Before(t) +} + +// IsDeprecated returns the cached deprecated status. +// The flag is computed when the PdscTag is added to a PidxXML (Read, AddPdsc, AddReplacePdsc) +// and is based on the reference date set at that time. +func (p *PdscTag) IsDeprecated() bool { + return p.isDeprecated +} diff --git a/cmd/xml/pidx_test.go b/cmd/xml/pidx_test.go index 9756defd..53ba630d 100644 --- a/cmd/xml/pidx_test.go +++ b/cmd/xml/pidx_test.go @@ -8,6 +8,7 @@ import ( "os" "strings" "testing" + "time" errs "github.com/open-cmsis-pack/cpackget/cmd/errors" "github.com/open-cmsis-pack/cpackget/cmd/utils" @@ -54,6 +55,48 @@ func TestPdscTag(t *testing.T) { assert.Contains(packURL, "1.2.3") assert.Equal("http://vendor.com/TheVendor.ThePack.1.2.3.pack", packURL) }) + + t.Run("test IsDeprecated with empty string", func(t *testing.T) { + pidx := xml.NewPidxXML("test.pidx", false) + assert.Nil(pidx.AddPdsc(xml.PdscTag{Vendor: "V", Name: "P", Version: "1.0.0"})) + tags := pidx.FindPdscTags(xml.PdscTag{Vendor: "V", Name: "P", Version: "1.0.0"}) + assert.Equal(1, len(tags)) + assert.False(tags[0].IsDeprecated()) + }) + + t.Run("test IsDeprecated with past date", func(t *testing.T) { + pidx := xml.NewPidxXML("test.pidx", false) + assert.Nil(pidx.AddPdsc(xml.PdscTag{Vendor: "V", Name: "P1", Version: "1.0.0", Deprecated: "2020-01-01"})) + tags := pidx.FindPdscTags(xml.PdscTag{Vendor: "V", Name: "P1", Version: "1.0.0"}) + assert.Equal(1, len(tags)) + assert.True(tags[0].IsDeprecated()) + }) + + t.Run("test IsDeprecated with today's date", func(t *testing.T) { + today := time.Now().Format("2006-01-02") + pidx := xml.NewPidxXML("test.pidx", false) + assert.Nil(pidx.AddPdsc(xml.PdscTag{Vendor: "V", Name: "P2", Version: "1.0.0", Deprecated: today})) + tags := pidx.FindPdscTags(xml.PdscTag{Vendor: "V", Name: "P2", Version: "1.0.0"}) + assert.Equal(1, len(tags)) + assert.True(tags[0].IsDeprecated()) + }) + + t.Run("test IsDeprecated with future date", func(t *testing.T) { + future := time.Now().AddDate(1, 0, 0).Format("2006-01-02") + pidx := xml.NewPidxXML("test.pidx", false) + assert.Nil(pidx.AddPdsc(xml.PdscTag{Vendor: "V", Name: "P3", Version: "1.0.0", Deprecated: future})) + tags := pidx.FindPdscTags(xml.PdscTag{Vendor: "V", Name: "P3", Version: "1.0.0"}) + assert.Equal(1, len(tags)) + assert.False(tags[0].IsDeprecated()) + }) + + t.Run("test IsDeprecated with invalid date", func(t *testing.T) { + pidx := xml.NewPidxXML("test.pidx", false) + assert.Nil(pidx.AddPdsc(xml.PdscTag{Vendor: "V", Name: "P4", Version: "1.0.0", Deprecated: "not-a-date"})) + tags := pidx.FindPdscTags(xml.PdscTag{Vendor: "V", Name: "P4", Version: "1.0.0"}) + assert.Equal(1, len(tags)) + assert.False(tags[0].IsDeprecated()) + }) } func TestPidxXML(t *testing.T) {