Merge branch 'master' into dpkg-compliant-version-compare

Author: neolynx

.github/workflows/golangci-lint.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Install and initialize swagger
         run: |
           go install github.com/swaggo/swag/cmd/swag@latest
-          swag init -q --markdownFiles docs
+          swag init -q --propertyStrategy pascalcase --markdownFiles docs
         shell: sh
 
       - name: golangci-lint

AUTHORS

@@ -75,3 +75,5 @@ List of contributors, in chronological order:
 * Agustin Henze (https://github.com/agustinhenze)
 * Tobias Assarsson (https://github.com/daedaluz)
 * Yaksh Bariya (https://github.com/thunder-coding)
+* Juan Calderon-Perez (https://github.com/gaby)
+* Ato Araki (https://github.com/atotto)

Makefile

@@ -77,7 +77,7 @@ azurite-stop:
 
 swagger: swagger-install
 	# Generate swagger docs
-	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+	@PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
 
 etcd-install:
 	# Install etcd
@@ -131,7 +131,7 @@ serve: prepare swagger-install  ## Run development server (auto recompiling)
 	test -f $(BINPATH)/air || go install github.com/air-verse/[email protected]
 	cp debian/aptly.conf ~/.aptly.conf
 	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
-	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
 dpkg: prepare swagger  ## Build debian packages
 	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)

api/mirror.go

@@ -343,6 +343,8 @@ type mirrorUpdateParams struct {
 	ForceUpdate bool `            json:"ForceUpdate"`
 	// Set "true" to skip downloading already downloaded packages
 	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+	// Set "true" to download only the latest version per package/architecture
+	LatestOnly bool `             json:"LatestOnly"`
 }
 
 // @Summary Update Mirror
@@ -434,7 +436,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

cmd/mirror_update.go

@@ -87,10 +87,11 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	)
 
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+	latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
 	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -292,6 +293,7 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
+	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")

completion.d/_aptly

@@ -211,6 +211,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-max-tries=[max download tries till process fails with download error]:number: " \
                             "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
+                            "-latest=[download only latest version of each package (per architecture)]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     rename)

completion.d/aptly

@@ -263,7 +263,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

deb/list.go

@@ -172,6 +172,39 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 	return err
 }
 
+// FilterLatest creates a copy of the package list containing only the
+// latest version for each package name/architecture pair.
+func (l *PackageList) FilterLatest() (*PackageList, error) {
+	if l == nil {
+		return nil, fmt.Errorf("package list is nil")
+	}
+
+	filtered := make(map[string]*Package, l.Len())
+
+	err := l.ForEach(func(p *Package) error {
+		key := p.Architecture + "|" + p.Name
+
+		if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 {
+			filtered[key] = p
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))
+
+	for _, pkg := range filtered {
+		if err = result.Add(pkg); err != nil {
+			return nil, err
+		}
+	}
+
+	return result, nil
+}
+
 // ForEachIndexed calls handler for each package in list in indexed order
 func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
 	if !l.indexed {

deb/list_test.go

@@ -503,3 +503,52 @@ func (s *PackageListSuite) TestArchitectures(c *C) {
 	sort.Strings(archs)
 	c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"})
 }
+
+func (s *PackageListSuite) TestFilterLatest(c *C) {
+	list := NewPackageList()
+
+	older := packageStanza.Copy()
+	older["Version"] = "1.0"
+	olderPkg := NewPackageFromControlFile(older)
+	_ = list.Add(olderPkg)
+
+	newer := packageStanza.Copy()
+	newer["Version"] = "2.0"
+	newerPkg := NewPackageFromControlFile(newer)
+	_ = list.Add(newerPkg)
+
+	shared := packageStanza.Copy()
+	shared["Architecture"] = ArchitectureAll
+	shared["Version"] = "3.0"
+	shared["Package"] = "shared"
+	sharedPkg := NewPackageFromControlFile(shared)
+	_ = list.Add(sharedPkg)
+
+	filtered, err := list.FilterLatest()
+	c.Assert(err, IsNil)
+	c.Assert(filtered.Len(), Equals, 2)
+	c.Check(filtered.Has(newerPkg), Equals, true)
+	c.Check(filtered.Has(sharedPkg), Equals, true)
+}
+
+func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) {
+	list := NewPackageListWithDuplicates(true, 2)
+
+	_ = list.Add(NewPackageFromControlFile(packageStanza.Copy()))
+
+	another := packageStanza.Copy()
+	another["Version"] = "7.41-1"
+	_ = list.Add(NewPackageFromControlFile(another))
+
+	filtered, err := list.FilterLatest()
+	c.Assert(err, IsNil)
+	c.Assert(filtered.duplicatesAllowed, Equals, true)
+}
+
+func (s *PackageListSuite) TestFilterLatestNil(c *C) {
+	var list *PackageList
+
+	filtered, err := list.FilterLatest()
+	c.Assert(err, ErrorMatches, "package list is nil")
+	c.Assert(filtered, IsNil)
+}

deb/remote.go

@@ -612,7 +612,19 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 }
 
 // BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+	if repo.packageList == nil {
+		err = fmt.Errorf("package list is empty, please (re)download package indexes")
+		return
+	}
+
+	if latestOnly {
+		repo.packageList, err = repo.packageList.FilterLatest()
+		if err != nil {
+			return
+		}
+	}
+
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
 	seen := make(map[string]int, repo.packageList.Len())