Skip to content

Commit 57eb9d0

Browse files
sschroewxiaoguang
sschroe
and
wxiaoguang
authored
Inherit submodules from template repository content (#16237)
Fix #10316 --------- Signed-off-by: Steffen Schröter <[email protected]> Co-authored-by: wxiaoguang <[email protected]>
1 parent 92a2900 commit 57eb9d0

File tree

17 files changed

+290
-136
lines changed

17 files changed

+290
-136
lines changed

‎modules/git/batch_reader.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -242,15 +242,15 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
242242
return out
243243
}
244244

245-
// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
245+
// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream
246246
// This carefully avoids allocations - except where fnameBuf is too small.
247247
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
248248
//
249249
// Each line is composed of:
250250
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <binary HASH>
251251
//
252252
// We don't attempt to convert the raw HASH to save a lot of time
253-
func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
253+
func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
254254
var readBytes []byte
255255

256256
// Read the Mode & fname
@@ -260,7 +260,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu
260260
}
261261
idx := bytes.IndexByte(readBytes, ' ')
262262
if idx < 0 {
263-
log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes)
263+
log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes)
264264
return mode, fname, sha, n, &ErrNotExist{}
265265
}
266266

‎modules/git/parse.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"bytes"
8+
"fmt"
9+
"strconv"
10+
"strings"
11+
12+
"code.gitea.io/gitea/modules/optional"
13+
)
14+
15+
var sepSpace = []byte{' '}
16+
17+
type LsTreeEntry struct {
18+
ID ObjectID
19+
EntryMode EntryMode
20+
Name string
21+
Size optional.Option[int64]
22+
}
23+
24+
func parseLsTreeLine(line []byte) (*LsTreeEntry, error) {
25+
// expect line to be of the form:
26+
// <mode> <type> <sha> <space-padded-size>\t<filename>
27+
// <mode> <type> <sha>\t<filename>
28+
29+
var err error
30+
posTab := bytes.IndexByte(line, '\t')
31+
if posTab == -1 {
32+
return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
33+
}
34+
35+
entry := new(LsTreeEntry)
36+
37+
entryAttrs := line[:posTab]
38+
entryName := line[posTab+1:]
39+
40+
entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
41+
_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
42+
entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
43+
if len(entryAttrs) > 0 {
44+
entrySize := entryAttrs // the last field is the space-padded-size
45+
size, _ := strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
46+
entry.Size = optional.Some(size)
47+
}
48+
49+
switch string(entryMode) {
50+
case "100644":
51+
entry.EntryMode = EntryModeBlob
52+
case "100755":
53+
entry.EntryMode = EntryModeExec
54+
case "120000":
55+
entry.EntryMode = EntryModeSymlink
56+
case "160000":
57+
entry.EntryMode = EntryModeCommit
58+
case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
59+
entry.EntryMode = EntryModeTree
60+
default:
61+
return nil, fmt.Errorf("unknown type: %v", string(entryMode))
62+
}
63+
64+
entry.ID, err = NewIDFromString(string(entryObjectID))
65+
if err != nil {
66+
return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
67+
}
68+
69+
if len(entryName) > 0 && entryName[0] == '"' {
70+
entry.Name, err = strconv.Unquote(string(entryName))
71+
if err != nil {
72+
return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
73+
}
74+
} else {
75+
entry.Name = string(entryName)
76+
}
77+
return entry, nil
78+
}

‎modules/git/parse_nogogit.go‎

Lines changed: 12 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ import (
1010
"bytes"
1111
"fmt"
1212
"io"
13-
"strconv"
14-
"strings"
1513

1614
"code.gitea.io/gitea/modules/log"
1715
)
@@ -21,71 +19,30 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
2119
return parseTreeEntries(data, nil)
2220
}
2321

24-
var sepSpace = []byte{' '}
25-
22+
// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory
2623
func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
27-
var err error
2824
entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
2925
for pos := 0; pos < len(data); {
30-
// expect line to be of the form:
31-
// <mode> <type> <sha> <space-padded-size>\t<filename>
32-
// <mode> <type> <sha>\t<filename>
3326
posEnd := bytes.IndexByte(data[pos:], '\n')
3427
if posEnd == -1 {
3528
posEnd = len(data)
3629
} else {
3730
posEnd += pos
3831
}
39-
line := data[pos:posEnd]
40-
posTab := bytes.IndexByte(line, '\t')
41-
if posTab == -1 {
42-
return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
43-
}
44-
45-
entry := new(TreeEntry)
46-
entry.ptree = ptree
47-
48-
entryAttrs := line[:posTab]
49-
entryName := line[posTab+1:]
50-
51-
entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
52-
_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
53-
entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
54-
if len(entryAttrs) > 0 {
55-
entrySize := entryAttrs // the last field is the space-padded-size
56-
entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
57-
entry.sized = true
58-
}
5932

60-
switch string(entryMode) {
61-
case "100644":
62-
entry.entryMode = EntryModeBlob
63-
case "100755":
64-
entry.entryMode = EntryModeExec
65-
case "120000":
66-
entry.entryMode = EntryModeSymlink
67-
case "160000":
68-
entry.entryMode = EntryModeCommit
69-
case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
70-
entry.entryMode = EntryModeTree
71-
default:
72-
return nil, fmt.Errorf("unknown type: %v", string(entryMode))
73-
}
74-
75-
entry.ID, err = NewIDFromString(string(entryObjectID))
33+
line := data[pos:posEnd]
34+
lsTreeLine, err := parseLsTreeLine(line)
7635
if err != nil {
77-
return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
36+
return nil, err
7837
}
79-
80-
if len(entryName) > 0 && entryName[0] == '"' {
81-
entry.name, err = strconv.Unquote(string(entryName))
82-
if err != nil {
83-
return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
84-
}
85-
} else {
86-
entry.name = string(entryName)
38+
entry := &TreeEntry{
39+
ptree: ptree,
40+
ID: lsTreeLine.ID,
41+
entryMode: lsTreeLine.EntryMode,
42+
name: lsTreeLine.Name,
43+
size: lsTreeLine.Size.Value(),
44+
sized: lsTreeLine.Size.Has(),
8745
}
88-
8946
pos = posEnd + 1
9047
entries = append(entries, entry)
9148
}
@@ -100,7 +57,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.
10057

10158
loop:
10259
for sz > 0 {
103-
mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
60+
mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
10461
if err != nil {
10562
if err == io.EOF {
10663
break loop

‎modules/git/pipeline/lfs_nogogit.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
114114
case "tree":
115115
var n int64
116116
for n < size {
117-
mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
117+
mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
118118
if err != nil {
119119
return nil, err
120120
}

‎modules/git/submodule.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"bufio"
8+
"context"
9+
"fmt"
10+
"os"
11+
12+
"code.gitea.io/gitea/modules/log"
13+
)
14+
15+
type TemplateSubmoduleCommit struct {
16+
Path string
17+
Commit string
18+
}
19+
20+
// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository
21+
// This function is only for generating new repos based on existing template, the template couldn't be too large.
22+
func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) {
23+
stdoutReader, stdoutWriter, err := os.Pipe()
24+
if err != nil {
25+
return nil, err
26+
}
27+
opts := &RunOpts{
28+
Dir: repoPath,
29+
Stdout: stdoutWriter,
30+
PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error {
31+
_ = stdoutWriter.Close()
32+
defer stdoutReader.Close()
33+
34+
scanner := bufio.NewScanner(stdoutReader)
35+
for scanner.Scan() {
36+
entry, err := parseLsTreeLine(scanner.Bytes())
37+
if err != nil {
38+
cancel()
39+
return err
40+
}
41+
if entry.EntryMode == EntryModeCommit {
42+
submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name, Commit: entry.ID.String()})
43+
}
44+
}
45+
return scanner.Err()
46+
},
47+
}
48+
err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts)
49+
if err != nil {
50+
return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err)
51+
}
52+
return submoduleCommits, nil
53+
}
54+
55+
// AddTemplateSubmoduleIndexes Adds the given submodules to the git index.
56+
// It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir.
57+
func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error {
58+
for _, submodule := range submodules {
59+
cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path)
60+
if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil {
61+
log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err)
62+
return err
63+
}
64+
}
65+
return nil
66+
}

‎modules/git/submodule_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package git
5+
6+
import (
7+
"context"
8+
"os"
9+
"path/filepath"
10+
"testing"
11+
12+
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
func TestGetTemplateSubmoduleCommits(t *testing.T) {
17+
testRepoPath := filepath.Join(testReposDir, "repo4_submodules")
18+
submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath)
19+
require.NoError(t, err)
20+
21+
assert.Len(t, submodules, 2)
22+
23+
assert.EqualValues(t, "<°)))><", submodules[0].Path)
24+
assert.EqualValues(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit)
25+
26+
assert.EqualValues(t, "libtest", submodules[1].Path)
27+
assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit)
28+
}
29+
30+
func TestAddTemplateSubmoduleIndexes(t *testing.T) {
31+
ctx := context.Background()
32+
tmpDir := t.TempDir()
33+
var err error
34+
_, _, err = NewCommand(ctx, "init").RunStdString(&RunOpts{Dir: tmpDir})
35+
require.NoError(t, err)
36+
_ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0o755)
37+
err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}})
38+
require.NoError(t, err)
39+
_, _, err = NewCommand(ctx, "add", "--all").RunStdString(&RunOpts{Dir: tmpDir})
40+
require.NoError(t, err)
41+
_, _, err = NewCommand(ctx, "-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(&RunOpts{Dir: tmpDir})
42+
require.NoError(t, err)
43+
submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir)
44+
require.NoError(t, err)
45+
assert.Len(t, submodules, 1)
46+
assert.EqualValues(t, "new-dir", submodules[0].Path)
47+
assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[0].Commit)
48+
}

‎modules/git/tests/repos/repo4_submodules/HEAD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ref: refs/heads/master

‎modules/git/tests/repos/repo4_submodules/config‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[core]
2+
repositoryformatversion = 0
3+
filemode = true
4+
bare = true

‎modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74‎

110 Bytes
Binary file not shown.

‎modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34‎

112 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)