Add support for repository names containing slashes.

Add support for different file compressions.
Prevent duplicates caused by different file compressions.

Co-authored-by: dragon <[email protected]>

Author: KN4CK3R

models/packages/package_file.go

@@ -221,6 +221,11 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag
 	return pfs, count, err
 }
 
+// HasFiles tests if there are files of packages matching the search options
+func HasFiles(ctx context.Context, opts *PackageFileSearchOptions) (bool, error) {
+	return db.Exist[PackageFile](ctx, opts.toConds())
+}
+
 // CalculateFileSize sums up all blob sizes matching the search options.
 // It does NOT respect the deduplication of blobs.
 func CalculateFileSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {

modules/packages/arch/metadata.go

@@ -6,6 +6,8 @@ package arch
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
+	"compress/gzip"
 	"io"
 	"regexp"
 	"strconv"
@@ -15,6 +17,7 @@ import (
 	"code.gitea.io/gitea/modules/validation"
 
 	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
 )
 
 const (
@@ -34,6 +37,7 @@ const (
 
 var (
 	ErrMissingPKGINFOFile  = util.NewInvalidArgumentErrorf(".PKGINFO file is missing")
+	ErrUnsupportedFormat   = util.NewInvalidArgumentErrorf("unsupported package container format")
 	ErrInvalidName         = util.NewInvalidArgumentErrorf("package name is invalid")
 	ErrInvalidVersion      = util.NewInvalidArgumentErrorf("package version is invalid")
 	ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid")
@@ -44,10 +48,11 @@ var (
 )
 
 type Package struct {
-	Name            string `json:"name"`
-	Version         string `json:"version"`
-	VersionMetadata VersionMetadata
-	FileMetadata    FileMetadata
+	Name                     string
+	Version                  string
+	VersionMetadata          VersionMetadata
+	FileMetadata             FileMetadata
+	FileCompressionExtension string
 }
 
 type VersionMetadata struct {
@@ -75,16 +80,50 @@ type FileMetadata struct {
 
 // ParsePackage parses an Arch package file
 func ParsePackage(r io.Reader) (*Package, error) {
-	zr, err := zstd.NewReader(r)
+	header := make([]byte, 10)
+	n, err := util.ReadAtMost(r, header)
 	if err != nil {
 		return nil, err
 	}
-	defer zr.Close()
+
+	r = io.MultiReader(bytes.NewReader(header[:n]), r)
+
+	var inner io.Reader
+	var compressionType string
+	if bytes.HasPrefix(header, []byte{0x28, 0xB5, 0x2F, 0xFD}) { // zst
+		zr, err := zstd.NewReader(r)
+		if err != nil {
+			return nil, err
+		}
+		defer zr.Close()
+
+		inner = zr
+		compressionType = "zst"
+	} else if bytes.HasPrefix(header, []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A}) { // xz
+		xzr, err := xz.NewReader(r)
+		if err != nil {
+			return nil, err
+		}
+
+		inner = xzr
+		compressionType = "xz"
+	} else if bytes.HasPrefix(header, []byte{0x1F, 0x8B}) { // gz
+		gzr, err := gzip.NewReader(r)
+		if err != nil {
+			return nil, err
+		}
+		defer gzr.Close()
+
+		inner = gzr
+		compressionType = "gz"
+	} else {
+		return nil, ErrUnsupportedFormat
+	}
 
 	var p *Package
 	files := make([]string, 0, 10)
 
-	tr := tar.NewReader(zr)
+	tr := tar.NewReader(inner)
 	for {
 		hd, err := tr.Next()
 		if err == io.EOF {
@@ -114,6 +153,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
 	}
 
 	p.FileMetadata.Files = files
+	p.FileCompressionExtension = compressionType
 
 	return p, nil
 }

modules/packages/arch/metadata_test.go

@@ -6,11 +6,13 @@ package arch
 import (
 	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"io"
 	"testing"
 
 	"github.com/klauspost/compress/zstd"
 	"github.com/stretchr/testify/assert"
+	"github.com/ulikunitz/xz"
 )
 
 const (
@@ -46,10 +48,18 @@ backup = usr/bin/paket1`)
 }
 
 func TestParsePackage(t *testing.T) {
-	createPackage := func(files map[string][]byte) io.Reader {
+	createPackage := func(compression string, files map[string][]byte) io.Reader {
 		var buf bytes.Buffer
-		zw, _ := zstd.NewWriter(&buf)
-		tw := tar.NewWriter(zw)
+		var cw io.WriteCloser
+		switch compression {
+		case "zst":
+			cw, _ = zstd.NewWriter(&buf)
+		case "xz":
+			cw, _ = xz.NewWriter(&buf)
+		case "gz":
+			cw = gzip.NewWriter(&buf)
+		}
+		tw := tar.NewWriter(cw)
 
 		for name, content := range files {
 			hdr := &tar.Header{
@@ -62,39 +72,43 @@ func TestParsePackage(t *testing.T) {
 		}
 
 		tw.Close()
-		zw.Close()
+		cw.Close()
 
 		return &buf
 	}
 
-	t.Run("MissingPKGINFOFile", func(t *testing.T) {
-		data := createPackage(map[string][]byte{"dummy.txt": {}})
+	for _, c := range []string{"gz", "xz", "zst"} {
+		t.Run(c, func(t *testing.T) {
+			t.Run("MissingPKGINFOFile", func(t *testing.T) {
+				data := createPackage(c, map[string][]byte{"dummy.txt": {}})
 
-		pp, err := ParsePackage(data)
-		assert.Nil(t, pp)
-		assert.ErrorIs(t, err, ErrMissingPKGINFOFile)
-	})
+				pp, err := ParsePackage(data)
+				assert.Nil(t, pp)
+				assert.ErrorIs(t, err, ErrMissingPKGINFOFile)
+			})
 
-	t.Run("InvalidPKGINFOFile", func(t *testing.T) {
-		data := createPackage(map[string][]byte{".PKGINFO": {}})
+			t.Run("InvalidPKGINFOFile", func(t *testing.T) {
+				data := createPackage(c, map[string][]byte{".PKGINFO": {}})
 
-		pp, err := ParsePackage(data)
-		assert.Nil(t, pp)
-		assert.ErrorIs(t, err, ErrInvalidName)
-	})
+				pp, err := ParsePackage(data)
+				assert.Nil(t, pp)
+				assert.ErrorIs(t, err, ErrInvalidName)
+			})
 
-	t.Run("Valid", func(t *testing.T) {
-		data := createPackage(map[string][]byte{
-			".PKGINFO":        createPKGINFOContent(packageName, packageVersion),
-			"/test/dummy.txt": {},
-		})
+			t.Run("Valid", func(t *testing.T) {
+				data := createPackage(c, map[string][]byte{
+					".PKGINFO":        createPKGINFOContent(packageName, packageVersion),
+					"/test/dummy.txt": {},
+				})
 
-		p, err := ParsePackage(data)
-		assert.NoError(t, err)
-		assert.NotNil(t, p)
+				p, err := ParsePackage(data)
+				assert.NoError(t, err)
+				assert.NotNil(t, p)
 
-		assert.ElementsMatch(t, []string{"/test/dummy.txt"}, p.FileMetadata.Files)
-	})
+				assert.ElementsMatch(t, []string{"/test/dummy.txt"}, p.FileMetadata.Files)
+			})
+		})
+	}
 }
 
 func TestParsePackageInfo(t *testing.T) {

routers/api/packages/api.go

@@ -137,15 +137,47 @@ func CommonRoutes() *web.Router {
 			})
 		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/arch", func() {
-			r.Get("/key", arch.GetRepositoryKey)
-			r.Group("/{repository}", func() {
-				r.Put("", reqPackageAccess(perm.AccessModeWrite), arch.UploadPackageFile)
-				r.Group("/{architecture}/{filename}", func() {
-					r.Get("", arch.DownloadPackageOrRepositoryFile)
-					r.Delete("", reqPackageAccess(perm.AccessModeWrite), arch.DeletePackageFile)
-				})
+			r.Methods("HEAD,GET", "/repository.key", arch.GetRepositoryKey)
+
+			r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) {
+				path := strings.Trim(ctx.PathParam("*"), "/")
+
+				if ctx.Req.Method == "PUT" {
+					reqPackageAccess(perm.AccessModeWrite)(ctx)
+					if ctx.Written() {
+						return
+					}
+					ctx.SetPathParam("repository", path)
+					arch.UploadPackageFile(ctx)
+					return
+				}
+
+				pathFields := strings.Split(path, "/")
+				pathFieldsLen := len(pathFields)
+
+				if pathFieldsLen >= 2 {
+					ctx.SetPathParam("repository", strings.Join(pathFields[:pathFieldsLen-2], "/"))
+					ctx.SetPathParam("architecture", pathFields[pathFieldsLen-2])
+					ctx.SetPathParam("filename", pathFields[pathFieldsLen-1])
+
+					if ctx.Req.Method == "HEAD" || ctx.Req.Method == "GET" {
+						arch.GetPackageOrRepositoryFile(ctx)
+						return
+					}
+
+					if ctx.Req.Method == "DELETE" {
+						reqPackageAccess(perm.AccessModeWrite)(ctx)
+						if ctx.Written() {
+							return
+						}
+						arch.DeletePackageFile(ctx)
+						return
+					}
+				}
+
+				ctx.Status(http.StatusNotFound)
 			})
-		})
+		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/cargo", func() {
 			r.Group("/api/v1/crates", func() {
 				r.Get("", cargo.SearchPackages)

routers/api/packages/arch/arch.go

@@ -42,7 +42,7 @@ func GetRepositoryKey(ctx *context.Context) {
 }
 
 func UploadPackageFile(ctx *context.Context) {
-	repository := strings.TrimSpace(ctx.Params("repository"))
+	repository := strings.TrimSpace(ctx.PathParam("repository"))
 	if repository == "" {
 		apiError(ctx, http.StatusBadRequest, "invalid repository")
 		return
@@ -96,6 +96,32 @@ func UploadPackageFile(ctx *context.Context) {
 		return
 	}
 
+	release, err := arch_service.AquireRegistryLock(ctx, ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer release()
+
+	// Search for duplicates with different file compression
+	has, err := packages_model.HasFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:     ctx.Package.Owner.ID,
+		PackageType: packages_model.TypeArch,
+		Query:       fmt.Sprintf("%s-%s-%s.pkg.tar.%%", pck.Name, pck.Version, pck.FileMetadata.Architecture),
+		Properties: map[string]string{
+			arch_module.PropertyRepository:   repository,
+			arch_module.PropertyArchitecture: pck.FileMetadata.Architecture,
+		},
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if has {
+		apiError(ctx, http.StatusConflict, packages_model.ErrDuplicatePackageFile)
+		return
+	}
+
 	_, _, err = packages_service.CreatePackageOrAddFileToExisting(
 		ctx,
 		&packages_service.PackageCreationInfo{
@@ -110,7 +136,7 @@ func UploadPackageFile(ctx *context.Context) {
 		},
 		&packages_service.PackageFileCreationInfo{
 			PackageFileInfo: packages_service.PackageFileInfo{
-				Filename:     fmt.Sprintf("%s-%s-%s.pck.tar.zst", pck.Name, pck.Version, pck.FileMetadata.Architecture),
+				Filename:     fmt.Sprintf("%s-%s-%s.pkg.tar.%s", pck.Name, pck.Version, pck.FileMetadata.Architecture, pck.FileCompressionExtension),
 				CompositeKey: fmt.Sprintf("%s|%s", repository, pck.FileMetadata.Architecture),
 			},
 			Creator: ctx.Doer,
@@ -144,10 +170,10 @@ func UploadPackageFile(ctx *context.Context) {
 	ctx.Status(http.StatusCreated)
 }
 
-func DownloadPackageOrRepositoryFile(ctx *context.Context) {
-	repository := ctx.Params("repository")
-	architecture := ctx.Params("architecture")
-	filename := ctx.Params("filename")
+func GetPackageOrRepositoryFile(ctx *context.Context) {
+	repository := ctx.PathParam("repository")
+	architecture := ctx.PathParam("architecture")
+	filename := ctx.PathParam("filename")
 	filenameOrig := filename
 
 	isSignature := strings.HasSuffix(filename, ".sig")
@@ -231,12 +257,19 @@ func DownloadPackageOrRepositoryFile(ctx *context.Context) {
 }
 
 func DeletePackageFile(ctx *context.Context) {
-	repository, architecture := ctx.Params("repository"), ctx.Params("architecture")
+	repository, architecture := ctx.PathParam("repository"), ctx.PathParam("architecture")
+
+	release, err := arch_service.AquireRegistryLock(ctx, ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer release()
 
 	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
 		OwnerID:      ctx.Package.Owner.ID,
 		PackageType:  packages_model.TypeArch,
-		Query:        ctx.Params("filename"),
+		Query:        ctx.PathParam("filename"),
 		CompositeKey: fmt.Sprintf("%s|%s", repository, architecture),
 	})
 	if err != nil {

routers/web/user/package.go

@@ -179,13 +179,13 @@ func ViewPackageVersion(ctx *context.Context) {
 	ctx.Data["IsPackagesPage"] = true
 	ctx.Data["PackageDescriptor"] = pd
 
+	registryHostURL, err := url.Parse(httplib.GuessCurrentHostURL(ctx))
+	if err != nil {
+		registryHostURL, _ = url.Parse(setting.AppURL)
+	}
+	ctx.Data["PackageRegistryHost"] = registryHostURL.Host
+
 	switch pd.Package.Type {
-	case packages_model.TypeContainer:
-		registryAppURL, err := url.Parse(httplib.GuessCurrentAppURL(ctx))
-		if err != nil {
-			registryAppURL, _ = url.Parse(setting.AppURL)
-		}
-		ctx.Data["RegistryHost"] = registryAppURL.Host
 	case packages_model.TypeAlpine:
 		branches := make(container.Set[string])
 		repositories := make(container.Set[string])
@@ -267,7 +267,6 @@ func ViewPackageVersion(ctx *context.Context) {
 	var (
 		total int64
 		pvs   []*packages_model.PackageVersion
-		err   error
 	)
 	switch pd.Package.Type {
 	case packages_model.TypeContainer:

services/forms/package_form.go

@@ -15,7 +15,7 @@ import (
 type PackageCleanupRuleForm struct {
 	ID            int64
 	Enabled       bool
-	Type          string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
+	Type          string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
 	KeepCount     int    `binding:"In(0,1,5,10,25,50,100)"`
 	KeepPattern   string `binding:"RegexPattern"`
 	RemoveDays    int    `binding:"In(0,7,14,30,60,90,180)"`