Merge branch 'main' into main

Author: bencurio

assets/go-licenses.json

@@ -1,10 +1,15 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
 //go:build ignore
 
 package main
 
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/md5"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -15,6 +20,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"code.gitea.io/gitea/build/license"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/util"
 )
 
@@ -77,7 +84,7 @@ func main() {
 	}
 
 	tr := tar.NewReader(gz)
-
+	aliasesFiles := make(map[string][]string)
 	for {
 		hdr, err := tr.Next()
 
@@ -97,26 +104,73 @@ func main() {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "README") {
+		fileBaseName := filepath.Base(hdr.Name)
+		licenseName := strings.TrimSuffix(fileBaseName, ".txt")
+
+		if strings.HasPrefix(fileBaseName, "README") {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") {
+		if strings.HasPrefix(fileBaseName, "deprecated_") {
 			continue
 		}
-		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt")))
+		out, err := os.Create(path.Join(destination, licenseName))
 		if err != nil {
 			log.Fatalf("Failed to create new file. %s", err)
 		}
 
 		defer out.Close()
 
-		if _, err := io.Copy(out, tr); err != nil {
+		// some license files have same content, so we need to detect these files and create a convert map into a json file
+		// Later we use this convert map to avoid adding same license content with different license name
+		h := md5.New()
+		// calculate md5 and write file in the same time
+		r := io.TeeReader(tr, h)
+		if _, err := io.Copy(out, r); err != nil {
 			log.Fatalf("Failed to write new file. %s", err)
 		} else {
 			fmt.Printf("Written %s\n", out.Name())
+
+			md5 := hex.EncodeToString(h.Sum(nil))
+			aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
 		}
 	}
 
+	// generate convert license name map
+	licenseAliases := make(map[string]string)
+	for _, fileNames := range aliasesFiles {
+		if len(fileNames) > 1 {
+			licenseName := license.GetLicenseNameFromAliases(fileNames)
+			if licenseName == "" {
+				// license name should not be empty as expected
+				// if it is empty, we need to rewrite the logic of GetLicenseNameFromAliases
+				log.Fatalf("GetLicenseNameFromAliases: license name is empty")
+			}
+			for _, fileName := range fileNames {
+				licenseAliases[fileName] = licenseName
+			}
+		}
+	}
+	// save convert license name map to file
+	b, err := json.Marshal(licenseAliases)
+	if err != nil {
+		log.Fatalf("Failed to create json bytes. %s", err)
+	}
+
+	licenseAliasesDestination := filepath.Join(destination, "etc", "license-aliases.json")
+	if err := os.MkdirAll(filepath.Dir(licenseAliasesDestination), 0o755); err != nil {
+		log.Fatalf("Failed to create directory for license aliases json file. %s", err)
+	}
+
+	f, err := os.Create(licenseAliasesDestination)
+	if err != nil {
+		log.Fatalf("Failed to create license aliases json file. %s", err)
+	}
+	defer f.Close()
+
+	if _, err = f.Write(b); err != nil {
+		log.Fatalf("Failed to write license aliases json file. %s", err)
+	}
+
 	fmt.Println("Done")
 }

build/generate-licenses.go

@@ -0,0 +1,41 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import "strings"
+
+func GetLicenseNameFromAliases(fnl []string) string {
+	if len(fnl) == 0 {
+		return ""
+	}
+
+	shortestItem := func(list []string) string {
+		s := list[0]
+		for _, l := range list[1:] {
+			if len(l) < len(s) {
+				s = l
+			}
+		}
+		return s
+	}
+	allHasPrefix := func(list []string, s string) bool {
+		for _, l := range list {
+			if !strings.HasPrefix(l, s) {
+				return false
+			}
+		}
+		return true
+	}
+
+	sl := shortestItem(fnl)
+	slv := strings.Split(sl, "-")
+	var result string
+	for i := len(slv); i >= 0; i-- {
+		result = strings.Join(slv[:i], "-")
+		if allHasPrefix(fnl, result) {
+			return result
+		}
+	}
+	return ""
+}

build/license/aliasgenerator.go

@@ -0,0 +1,39 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetLicenseNameFromAliases(t *testing.T) {
+	tests := []struct {
+		target string
+		inputs []string
+	}{
+		{
+			// real case which you can find in license-aliases.json
+			target: "AGPL-1.0",
+			inputs: []string{
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+		{
+			target: "",
+			inputs: []string{
+				"APSL-1.0",
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		result := GetLicenseNameFromAliases(tt.inputs)
+		assert.Equal(t, result, tt.target)
+	}
+}

build/license/aliasgenerator_test.go

@@ -20,8 +20,10 @@ import (
 	asymkey_model "code.gitea.io/gitea/models/asymkey"
 	git_model "code.gitea.io/gitea/models/git"
 	"code.gitea.io/gitea/models/perm"
+	"code.gitea.io/gitea/modules/container"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/lfstransfer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/pprof"
 	"code.gitea.io/gitea/modules/private"
@@ -36,7 +38,11 @@ import (
 )
 
 const (
-	lfsAuthenticateVerb = "git-lfs-authenticate"
+	verbUploadPack      = "git-upload-pack"
+	verbUploadArchive   = "git-upload-archive"
+	verbReceivePack     = "git-receive-pack"
+	verbLfsAuthenticate = "git-lfs-authenticate"
+	verbLfsTransfer     = "git-lfs-transfer"
 )
 
 // CmdServ represents the available serv sub-command.
@@ -73,12 +79,18 @@ func setup(ctx context.Context, debug bool) {
 }
 
 var (
-	allowedCommands = map[string]perm.AccessMode{
-		"git-upload-pack":    perm.AccessModeRead,
-		"git-upload-archive": perm.AccessModeRead,
-		"git-receive-pack":   perm.AccessModeWrite,
-		lfsAuthenticateVerb:  perm.AccessModeNone,
-	}
+	// keep getAccessMode() in sync
+	allowedCommands = container.SetOf(
+		verbUploadPack,
+		verbUploadArchive,
+		verbReceivePack,
+		verbLfsAuthenticate,
+		verbLfsTransfer,
+	)
+	allowedCommandsLfs = container.SetOf(
+		verbLfsAuthenticate,
+		verbLfsTransfer,
+	)
 	alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`)
 )
 
@@ -124,6 +136,45 @@ func handleCliResponseExtra(extra private.ResponseExtra) error {
 	return nil
 }
 
+func getAccessMode(verb, lfsVerb string) perm.AccessMode {
+	switch verb {
+	case verbUploadPack, verbUploadArchive:
+		return perm.AccessModeRead
+	case verbReceivePack:
+		return perm.AccessModeWrite
+	case verbLfsAuthenticate, verbLfsTransfer:
+		switch lfsVerb {
+		case "upload":
+			return perm.AccessModeWrite
+		case "download":
+			return perm.AccessModeRead
+		}
+	}
+	// should be unreachable
+	return perm.AccessModeNone
+}
+
+func getLFSAuthToken(ctx context.Context, lfsVerb string, results *private.ServCommandResults) (string, error) {
+	now := time.Now()
+	claims := lfs.Claims{
+		RegisteredClaims: jwt.RegisteredClaims{
+			ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)),
+			NotBefore: jwt.NewNumericDate(now),
+		},
+		RepoID: results.RepoID,
+		Op:     lfsVerb,
+		UserID: results.UserID,
+	}
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+
+	// Sign and get the complete encoded token as a string using the secret
+	tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes)
+	if err != nil {
+		return "", fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err)
+	}
+	return fmt.Sprintf("Bearer %s", tokenString), nil
+}
+
 func runServ(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
@@ -198,15 +249,6 @@ func runServ(c *cli.Context) error {
 	repoPath := strings.TrimPrefix(words[1], "/")
 
 	var lfsVerb string
-	if verb == lfsAuthenticateVerb {
-		if !setting.LFS.StartServer {
-			return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
-		}
-
-		if len(words) > 2 {
-			lfsVerb = words[2]
-		}
-	}
 
 	rr := strings.SplitN(repoPath, "/", 2)
 	if len(rr) != 2 {
@@ -243,53 +285,52 @@ func runServ(c *cli.Context) error {
 		}()
 	}
 
-	requestedMode, has := allowedCommands[verb]
-	if !has {
+	if allowedCommands.Contains(verb) {
+		if allowedCommandsLfs.Contains(verb) {
+			if !setting.LFS.StartServer {
+				return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
+			}
+			if verb == verbLfsTransfer && !setting.LFS.AllowPureSSH {
+				return fail(ctx, "Unknown git command", "LFS SSH transfer connection denied, pure SSH protocol is disabled")
+			}
+			if len(words) > 2 {
+				lfsVerb = words[2]
+			}
+		}
+	} else {
 		return fail(ctx, "Unknown git command", "Unknown git command %s", verb)
 	}
 
-	if verb == lfsAuthenticateVerb {
-		if lfsVerb == "upload" {
-			requestedMode = perm.AccessModeWrite
-		} else if lfsVerb == "download" {
-			requestedMode = perm.AccessModeRead
-		} else {
-			return fail(ctx, "Unknown LFS verb", "Unknown lfs verb %s", lfsVerb)
-		}
-	}
+	requestedMode := getAccessMode(verb, lfsVerb)
 
 	results, extra := private.ServCommand(ctx, keyID, username, reponame, requestedMode, verb, lfsVerb)
 	if extra.HasError() {
 		return fail(ctx, extra.UserMsg, "ServCommand failed: %s", extra.Error)
 	}
 
+	// LFS SSH protocol
+	if verb == verbLfsTransfer {
+		token, err := getLFSAuthToken(ctx, lfsVerb, results)
+		if err != nil {
+			return err
+		}
+		return lfstransfer.Main(ctx, repoPath, lfsVerb, token)
+	}
+
 	// LFS token authentication
-	if verb == lfsAuthenticateVerb {
+	if verb == verbLfsAuthenticate {
 		url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName))
 
-		now := time.Now()
-		claims := lfs.Claims{
-			RegisteredClaims: jwt.RegisteredClaims{
-				ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)),
-				NotBefore: jwt.NewNumericDate(now),
-			},
-			RepoID: results.RepoID,
-			Op:     lfsVerb,
-			UserID: results.UserID,
-		}
-		token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
-
-		// Sign and get the complete encoded token as a string using the secret
-		tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes)
+		token, err := getLFSAuthToken(ctx, lfsVerb, results)
 		if err != nil {
-			return fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err)
+			return err
 		}
 
 		tokenAuthentication := &git_model.LFSTokenResponse{
 			Header: make(map[string]string),
 			Href:   url,
 		}
-		tokenAuthentication.Header["Authorization"] = fmt.Sprintf("Bearer %s", tokenString)
+		tokenAuthentication.Header["Authorization"] = token
 
 		enc := json.NewEncoder(os.Stdout)
 		err = enc.Encode(tokenAuthentication)

cmd/serv.go

@@ -306,6 +306,8 @@ RUN_USER = ; git
 ;; Enables git-lfs support. true or false, default is false.
 ;LFS_START_SERVER = false
 ;;
+;; Enables git-lfs SSH protocol support. true or false, default is false.
+;LFS_ALLOW_PURE_SSH = false
 ;;
 ;; LFS authentication secret, change this yourself
 ;LFS_JWT_SECRET =