Merge branch 'main' into lunny/issue_dev

Author: lunny

assets/go-licenses.json

@@ -1,10 +1,15 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
 //go:build ignore
 
 package main
 
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/md5"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -15,6 +20,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"code.gitea.io/gitea/build/license"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/util"
 )
 
@@ -77,7 +84,7 @@ func main() {
 	}
 
 	tr := tar.NewReader(gz)
-
+	aliasesFiles := make(map[string][]string)
 	for {
 		hdr, err := tr.Next()
 
@@ -97,26 +104,73 @@ func main() {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "README") {
+		fileBaseName := filepath.Base(hdr.Name)
+		licenseName := strings.TrimSuffix(fileBaseName, ".txt")
+
+		if strings.HasPrefix(fileBaseName, "README") {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") {
+		if strings.HasPrefix(fileBaseName, "deprecated_") {
 			continue
 		}
-		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt")))
+		out, err := os.Create(path.Join(destination, licenseName))
 		if err != nil {
 			log.Fatalf("Failed to create new file. %s", err)
 		}
 
 		defer out.Close()
 
-		if _, err := io.Copy(out, tr); err != nil {
+		// some license files have same content, so we need to detect these files and create a convert map into a json file
+		// Later we use this convert map to avoid adding same license content with different license name
+		h := md5.New()
+		// calculate md5 and write file in the same time
+		r := io.TeeReader(tr, h)
+		if _, err := io.Copy(out, r); err != nil {
 			log.Fatalf("Failed to write new file. %s", err)
 		} else {
 			fmt.Printf("Written %s\n", out.Name())
+
+			md5 := hex.EncodeToString(h.Sum(nil))
+			aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
 		}
 	}
 
+	// generate convert license name map
+	licenseAliases := make(map[string]string)
+	for _, fileNames := range aliasesFiles {
+		if len(fileNames) > 1 {
+			licenseName := license.GetLicenseNameFromAliases(fileNames)
+			if licenseName == "" {
+				// license name should not be empty as expected
+				// if it is empty, we need to rewrite the logic of GetLicenseNameFromAliases
+				log.Fatalf("GetLicenseNameFromAliases: license name is empty")
+			}
+			for _, fileName := range fileNames {
+				licenseAliases[fileName] = licenseName
+			}
+		}
+	}
+	// save convert license name map to file
+	b, err := json.Marshal(licenseAliases)
+	if err != nil {
+		log.Fatalf("Failed to create json bytes. %s", err)
+	}
+
+	licenseAliasesDestination := filepath.Join(destination, "etc", "license-aliases.json")
+	if err := os.MkdirAll(filepath.Dir(licenseAliasesDestination), 0o755); err != nil {
+		log.Fatalf("Failed to create directory for license aliases json file. %s", err)
+	}
+
+	f, err := os.Create(licenseAliasesDestination)
+	if err != nil {
+		log.Fatalf("Failed to create license aliases json file. %s", err)
+	}
+	defer f.Close()
+
+	if _, err = f.Write(b); err != nil {
+		log.Fatalf("Failed to write license aliases json file. %s", err)
+	}
+
 	fmt.Println("Done")
 }

build/generate-licenses.go

@@ -0,0 +1,41 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import "strings"
+
+func GetLicenseNameFromAliases(fnl []string) string {
+	if len(fnl) == 0 {
+		return ""
+	}
+
+	shortestItem := func(list []string) string {
+		s := list[0]
+		for _, l := range list[1:] {
+			if len(l) < len(s) {
+				s = l
+			}
+		}
+		return s
+	}
+	allHasPrefix := func(list []string, s string) bool {
+		for _, l := range list {
+			if !strings.HasPrefix(l, s) {
+				return false
+			}
+		}
+		return true
+	}
+
+	sl := shortestItem(fnl)
+	slv := strings.Split(sl, "-")
+	var result string
+	for i := len(slv); i >= 0; i-- {
+		result = strings.Join(slv[:i], "-")
+		if allHasPrefix(fnl, result) {
+			return result
+		}
+	}
+	return ""
+}

build/license/aliasgenerator.go

@@ -0,0 +1,39 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetLicenseNameFromAliases(t *testing.T) {
+	tests := []struct {
+		target string
+		inputs []string
+	}{
+		{
+			// real case which you can find in license-aliases.json
+			target: "AGPL-1.0",
+			inputs: []string{
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+		{
+			target: "",
+			inputs: []string{
+				"APSL-1.0",
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		result := GetLicenseNameFromAliases(tt.inputs)
+		assert.Equal(t, result, tt.target)
+	}
+}

build/license/aliasgenerator_test.go

@@ -68,6 +68,7 @@ require (
 	github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/go-github/v61 v61.0.0
+	github.com/google/licenseclassifier/v2 v2.0.0
 	github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/feeds v1.2.0

go.mod

@@ -441,6 +441,8 @@ github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=
+github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM=
 github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 h1:ASJ/LAqdCHOyMYI+dwNxn7Rd8FscNkMyTr1KZU1JI/M=
 github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
@@ -735,6 +737,7 @@ github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jN
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

go.sum

@@ -83,3 +83,22 @@
   issue_id: 2 # in repo_id 1
   review_id: 20
   created_unix: 946684810
+
+-
+  id: 10
+  type: 22 # review
+  poster_id: 5
+  issue_id: 3 # in repo_id 1
+  content: "reviewed by user5"
+  review_id: 21
+  created_unix: 946684816
+
+-
+  id: 11
+  type: 27 # review request
+  poster_id: 2
+  issue_id: 3 # in repo_id 1
+  content: "review request for user5"
+  review_id: 22
+  assignee_id: 5
+  created_unix: 946684817

models/fixtures/comment.yml

@@ -0,0 +1 @@
+[] # empty

models/fixtures/repo_license.yml

@@ -26,7 +26,7 @@
   fork_id: 0
   is_template: false
   template_id: 0
-  size: 7597
+  size: 8478
   is_fsck_enabled: true
   close_issues_via_commit_in_any_branch: false
 

models/fixtures/repository.yml

@@ -179,3 +179,22 @@
   content: "Review Comment"
   updated_unix: 946684810
   created_unix: 946684810
+
+-
+  id: 21
+  type: 2
+  reviewer_id: 5
+  issue_id: 3
+  content: "reviewed by user5"
+  commit_id: 4a357436d925b5c974181ff12a994538ddc5a269
+  updated_unix: 946684816
+  created_unix: 946684816
+
+-
+  id: 22
+  type: 4
+  reviewer_id: 5
+  issue_id: 3
+  content: "review request for user5"
+  updated_unix: 946684817
+  created_unix: 946684817