Skip to content

Commit 0a4479c

Browse files
rossigeerossigee
committed
Add build tools for code processing and testing
1 parent 49b6761 commit 0a4479c

File tree

3 files changed

+411
-0
lines changed

3 files changed

+411
-0
lines changed

‎tools/code-batch-process.go‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
// Copyright 2021 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
//go:build ignore
5+
6+
package main
7+
8+
import (
9+
"fmt"
10+
"log"
11+
"os"
12+
"os/exec"
13+
"path/filepath"
14+
"regexp"
15+
"slices"
16+
"strconv"
17+
"strings"
18+
19+
"code.gitea.io/gitea/build/codeformat"
20+
)
21+
22+
// Windows has a limitation for command line arguments, the size can not exceed 32KB.
23+
// So we have to feed the files to some tools (like gofmt) batch by batch
24+
25+
// We also introduce a `gitea-fmt` command, it does better import formatting than gofmt/goimports. `gitea-fmt` calls `gofmt` internally.
26+
27+
var optionLogVerbose bool
28+
29+
func logVerbose(msg string, args ...any) {
30+
if optionLogVerbose {
31+
log.Printf(msg, args...)
32+
}
33+
}
34+
35+
func passThroughCmd(cmd string, args []string) error {
36+
foundCmd, err := exec.LookPath(cmd)
37+
if err != nil {
38+
log.Fatalf("can not find cmd: %s", cmd)
39+
}
40+
c := exec.Cmd{
41+
Path: foundCmd,
42+
Args: append([]string{cmd}, args...),
43+
Stdin: os.Stdin,
44+
Stdout: os.Stdout,
45+
Stderr: os.Stderr,
46+
}
47+
return c.Run()
48+
}
49+
50+
type fileCollector struct {
51+
dirs []string
52+
includePatterns []*regexp.Regexp
53+
excludePatterns []*regexp.Regexp
54+
batchSize int
55+
}
56+
57+
func newFileCollector(fileFilter string, batchSize int) (*fileCollector, error) {
58+
co := &fileCollector{batchSize: batchSize}
59+
if fileFilter == "go-own" {
60+
co.dirs = []string{
61+
"build",
62+
"cmd",
63+
"contrib",
64+
"tests",
65+
"models",
66+
"modules",
67+
"routers",
68+
"services",
69+
}
70+
co.includePatterns = append(co.includePatterns, regexp.MustCompile(`.*\.go$`))
71+
72+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`.*\bbindata\.go$`))
73+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`\.pb\.go$`))
74+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/gitea-repositories-meta`))
75+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/integration/migration-test`))
76+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`modules/git/tests`))
77+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/fixtures`))
78+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/migrations/fixtures`))
79+
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`services/gitdiff/testdata`))
80+
}
81+
82+
if co.dirs == nil {
83+
return nil, fmt.Errorf("unknown file-filter: %s", fileFilter)
84+
}
85+
return co, nil
86+
}
87+
88+
func (fc *fileCollector) matchPatterns(path string, regexps []*regexp.Regexp) bool {
89+
path = strings.ReplaceAll(path, "\\", "/")
90+
for _, re := range regexps {
91+
if re.MatchString(path) {
92+
return true
93+
}
94+
}
95+
return false
96+
}
97+
98+
func (fc *fileCollector) collectFiles() (res [][]string, err error) {
99+
var batch []string
100+
for _, dir := range fc.dirs {
101+
err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
102+
include := len(fc.includePatterns) == 0 || fc.matchPatterns(path, fc.includePatterns)
103+
exclude := fc.matchPatterns(path, fc.excludePatterns)
104+
process := include && !exclude
105+
if !process {
106+
if d.IsDir() {
107+
if exclude {
108+
logVerbose("exclude dir %s", path)
109+
return filepath.SkipDir
110+
}
111+
// for a directory, if it is not excluded explicitly, we should walk into
112+
return nil
113+
}
114+
// for a file, we skip it if it shouldn't be processed
115+
logVerbose("skip process %s", path)
116+
return nil
117+
}
118+
if d.IsDir() {
119+
// skip dir, we don't add dirs to the file list now
120+
return nil
121+
}
122+
if len(batch) >= fc.batchSize {
123+
res = append(res, batch)
124+
batch = nil
125+
}
126+
batch = append(batch, path)
127+
return nil
128+
})
129+
if err != nil {
130+
return nil, err
131+
}
132+
}
133+
res = append(res, batch)
134+
return res, nil
135+
}
136+
137+
// substArgFiles expands the {file-list} to a real file list for commands
138+
func substArgFiles(args, files []string) []string {
139+
for i, s := range args {
140+
if s == "{file-list}" {
141+
newArgs := append(args[:i], files...)
142+
newArgs = append(newArgs, args[i+1:]...)
143+
return newArgs
144+
}
145+
}
146+
return args
147+
}
148+
149+
func exitWithCmdErrors(subCmd string, subArgs []string, cmdErrors []error) {
150+
for _, err := range cmdErrors {
151+
if err != nil {
152+
if exitError, ok := err.(*exec.ExitError); ok {
153+
exitCode := exitError.ExitCode()
154+
log.Printf("run command failed (code=%d): %s %v", exitCode, subCmd, subArgs)
155+
os.Exit(exitCode)
156+
} else {
157+
log.Fatalf("run command failed (err=%s) %s %v", err, subCmd, subArgs)
158+
}
159+
}
160+
}
161+
}
162+
163+
func parseArgs() (mainOptions map[string]string, subCmd string, subArgs []string) {
164+
mainOptions = map[string]string{}
165+
for i := 1; i < len(os.Args); i++ {
166+
arg := os.Args[i]
167+
if arg == "" {
168+
break
169+
}
170+
if arg[0] == '-' {
171+
arg = strings.TrimPrefix(arg, "-")
172+
arg = strings.TrimPrefix(arg, "-")
173+
fields := strings.SplitN(arg, "=", 2)
174+
if len(fields) == 1 {
175+
mainOptions[fields[0]] = "1"
176+
} else {
177+
mainOptions[fields[0]] = fields[1]
178+
}
179+
} else {
180+
subCmd = arg
181+
subArgs = os.Args[i+1:]
182+
break
183+
}
184+
}
185+
return mainOptions, subCmd, subArgs
186+
}
187+
188+
func showUsage() {
189+
fmt.Printf(`Usage: %[1]s [options] {command} [arguments]
190+
191+
Options:
192+
--verbose
193+
--file-filter=go-own
194+
--batch-size=100
195+
196+
Commands:
197+
%[1]s gofmt ...
198+
199+
Arguments:
200+
{file-list} the file list
201+
202+
Example:
203+
%[1]s gofmt -s -d {file-list}
204+
205+
`, "file-batch-exec")
206+
}
207+
208+
func newFileCollectorFromMainOptions(mainOptions map[string]string) (fc *fileCollector, err error) {
209+
fileFilter := mainOptions["file-filter"]
210+
if fileFilter == "" {
211+
fileFilter = "go-own"
212+
}
213+
batchSize, _ := strconv.Atoi(mainOptions["batch-size"])
214+
if batchSize == 0 {
215+
batchSize = 100
216+
}
217+
218+
return newFileCollector(fileFilter, batchSize)
219+
}
220+
221+
func giteaFormatGoImports(files []string, doWriteFile bool) error {
222+
for _, file := range files {
223+
if err := codeformat.FormatGoImports(file, doWriteFile); err != nil {
224+
log.Printf("failed to format go imports: %s, err=%v", file, err)
225+
return err
226+
}
227+
}
228+
return nil
229+
}
230+
231+
func main() {
232+
mainOptions, subCmd, subArgs := parseArgs()
233+
if subCmd == "" {
234+
showUsage()
235+
os.Exit(1)
236+
}
237+
optionLogVerbose = mainOptions["verbose"] != ""
238+
239+
fc, err := newFileCollectorFromMainOptions(mainOptions)
240+
if err != nil {
241+
log.Fatalf("can not create file collector: %s", err.Error())
242+
}
243+
244+
fileBatches, err := fc.collectFiles()
245+
if err != nil {
246+
log.Fatalf("can not collect files: %s", err.Error())
247+
}
248+
249+
processed := 0
250+
var cmdErrors []error
251+
for _, files := range fileBatches {
252+
if len(files) == 0 {
253+
break
254+
}
255+
substArgs := substArgFiles(subArgs, files)
256+
logVerbose("batch cmd: %s %v", subCmd, substArgs)
257+
switch subCmd {
258+
case "gitea-fmt":
259+
if slices.Contains(subArgs, "-d") {
260+
log.Print("the -d option is not supported by gitea-fmt")
261+
}
262+
cmdErrors = append(cmdErrors, giteaFormatGoImports(files, slices.Contains(subArgs, "-w")))
263+
cmdErrors = append(cmdErrors, passThroughCmd("gofmt", append([]string{"-w", "-r", "interface{} -> any"}, substArgs...)))
264+
cmdErrors = append(cmdErrors, passThroughCmd("go", append([]string{"run", os.Getenv("GOFUMPT_PACKAGE"), "-extra"}, substArgs...)))
265+
default:
266+
log.Fatalf("unknown cmd: %s %v", subCmd, subArgs)
267+
}
268+
processed += len(files)
269+
}
270+
271+
logVerbose("processed %d files", processed)
272+
exitWithCmdErrors(subCmd, subArgs, cmdErrors)
273+
}

‎tools/gocovmerge.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Copyright (c) 2015, Wade Simmons
3+
// SPDX-License-Identifier: MIT
4+
5+
// gocovmerge takes the results from multiple `go test -coverprofile` runs and
6+
// merges them into one profile
7+
8+
//go:build ignore
9+
10+
package main
11+
12+
import (
13+
"flag"
14+
"fmt"
15+
"io"
16+
"log"
17+
"os"
18+
"sort"
19+
20+
"golang.org/x/tools/cover"
21+
)
22+
23+
func mergeProfiles(p, merge *cover.Profile) {
24+
if p.Mode != merge.Mode {
25+
log.Fatalf("cannot merge profiles with different modes")
26+
}
27+
// Since the blocks are sorted, we can keep track of where the last block
28+
// was inserted and only look at the blocks after that as targets for merge
29+
startIndex := 0
30+
for _, b := range merge.Blocks {
31+
startIndex = mergeProfileBlock(p, b, startIndex)
32+
}
33+
}
34+
35+
func mergeProfileBlock(p *cover.Profile, pb cover.ProfileBlock, startIndex int) int {
36+
sortFunc := func(i int) bool {
37+
pi := p.Blocks[i+startIndex]
38+
return pi.StartLine >= pb.StartLine && (pi.StartLine != pb.StartLine || pi.StartCol >= pb.StartCol)
39+
}
40+
41+
i := 0
42+
if sortFunc(i) != true {
43+
i = sort.Search(len(p.Blocks)-startIndex, sortFunc)
44+
}
45+
i += startIndex
46+
if i < len(p.Blocks) && p.Blocks[i].StartLine == pb.StartLine && p.Blocks[i].StartCol == pb.StartCol {
47+
if p.Blocks[i].EndLine != pb.EndLine || p.Blocks[i].EndCol != pb.EndCol {
48+
log.Fatalf("OVERLAP MERGE: %v %v %v", p.FileName, p.Blocks[i], pb)
49+
}
50+
switch p.Mode {
51+
case "set":
52+
p.Blocks[i].Count |= pb.Count
53+
case "count", "atomic":
54+
p.Blocks[i].Count += pb.Count
55+
default:
56+
log.Fatalf("unsupported covermode: '%s'", p.Mode)
57+
}
58+
} else {
59+
if i > 0 {
60+
pa := p.Blocks[i-1]
61+
if pa.EndLine >= pb.EndLine && (pa.EndLine != pb.EndLine || pa.EndCol > pb.EndCol) {
62+
log.Fatalf("OVERLAP BEFORE: %v %v %v", p.FileName, pa, pb)
63+
}
64+
}
65+
if i < len(p.Blocks)-1 {
66+
pa := p.Blocks[i+1]
67+
if pa.StartLine <= pb.StartLine && (pa.StartLine != pb.StartLine || pa.StartCol < pb.StartCol) {
68+
log.Fatalf("OVERLAP AFTER: %v %v %v", p.FileName, pa, pb)
69+
}
70+
}
71+
p.Blocks = append(p.Blocks, cover.ProfileBlock{})
72+
copy(p.Blocks[i+1:], p.Blocks[i:])
73+
p.Blocks[i] = pb
74+
}
75+
return i + 1
76+
}
77+
78+
func addProfile(profiles []*cover.Profile, p *cover.Profile) []*cover.Profile {
79+
i := sort.Search(len(profiles), func(i int) bool { return profiles[i].FileName >= p.FileName })
80+
if i < len(profiles) && profiles[i].FileName == p.FileName {
81+
mergeProfiles(profiles[i], p)
82+
} else {
83+
profiles = append(profiles, nil)
84+
copy(profiles[i+1:], profiles[i:])
85+
profiles[i] = p
86+
}
87+
return profiles
88+
}
89+
90+
func dumpProfiles(profiles []*cover.Profile, out io.Writer) {
91+
if len(profiles) == 0 {
92+
return
93+
}
94+
fmt.Fprintf(out, "mode: %s\n", profiles[0].Mode)
95+
for _, p := range profiles {
96+
for _, b := range p.Blocks {
97+
fmt.Fprintf(out, "%s:%d.%d,%d.%d %d %d\n", p.FileName, b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt, b.Count)
98+
}
99+
}
100+
}
101+
102+
func main() {
103+
flag.Parse()
104+
105+
var merged []*cover.Profile
106+
107+
for _, file := range flag.Args() {
108+
profiles, err := cover.ParseProfiles(file)
109+
if err != nil {
110+
log.Fatalf("failed to parse profile '%s': %v", file, err)
111+
}
112+
for _, p := range profiles {
113+
merged = addProfile(merged, p)
114+
}
115+
}
116+
117+
dumpProfiles(merged, os.Stdout)
118+
}

0 commit comments

Comments
 (0)