Skip to content

Commit c934b90

Browse files
committed
Merge branch 'buildout' into 'v3'
fix base buildout See merge request develop/client!7
2 parents f7dc5d2 + 5dfafcb commit c934b90

File tree

4 files changed

+193
-15
lines changed

4 files changed

+193
-15
lines changed

go.mod

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ require (
3333
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
3434
golang.org/x/mod v0.18.0
3535
golang.org/x/net v0.26.0
36-
golang.org/x/sync v0.10.0
37-
golang.org/x/sys v0.28.0
38-
golang.org/x/text v0.21.0
36+
golang.org/x/sync v0.7.0
37+
golang.org/x/sys v0.24.0
38+
golang.org/x/text v0.16.0
3939
gopkg.in/yaml.v3 v3.0.1
4040
)
4141

@@ -77,10 +77,11 @@ require (
7777
github.com/xanzy/go-gitlab v0.93.1 // indirect
7878
github.com/xanzy/ssh-agent v0.3.3 // indirect
7979
go.uber.org/multierr v1.11.0 // indirect
80-
golang.org/x/crypto v0.31.0 // indirect
80+
golang.org/x/crypto v0.24.0 // indirect
8181
golang.org/x/oauth2 v0.21.0 // indirect
82-
golang.org/x/term v0.27.0 // indirect
82+
golang.org/x/term v0.21.0 // indirect
8383
golang.org/x/time v0.5.0 // indirect
84+
gopkg.in/ini.v1 v1.67.0 // indirect
8485
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
8586
gopkg.in/warnings.v0 v0.1.2 // indirect
8687
)

go.sum

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -213,8 +213,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
213213
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
214214
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
215215
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
216-
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
217-
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
216+
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
217+
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
218218
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
219219
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
220220
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -238,8 +238,8 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
238238
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
239239
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
240240
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
241-
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
242-
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
241+
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
242+
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
243243
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
244244
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
245245
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -258,15 +258,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
258258
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
259259
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
260260
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
261-
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
262-
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
261+
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
262+
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
263263
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
264264
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
265265
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
266266
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
267267
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
268-
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
269-
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
268+
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
269+
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
270270
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
271271
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
272272
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -275,8 +275,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
275275
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
276276
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
277277
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
278-
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
279-
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
278+
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
279+
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
280280
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
281281
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
282282
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -291,6 +291,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
291291
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
292292
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
293293
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
294+
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
295+
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
294296
gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
295297
gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
296298
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=

module/python/buildout/buildout.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ func InspectProject(ctx context.Context, dir string) (*model.Module, error) {
117117
}
118118
}
119119
var comps = make(map[[2]string]struct{})
120+
MetadataComps := make(map[string]string)
121+
BuildoutCfgComps := make(map[string]string)
120122
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, e error) error {
121123
if ctx.Err() != nil {
122124
return ctx.Err()
@@ -134,9 +136,22 @@ func InspectProject(ctx context.Context, dir string) (*model.Module, error) {
134136
return nil
135137
}
136138
comps[[2]string{n, v}] = struct{}{}
139+
MetadataComps[n] = v
140+
}
141+
if d.Name() == "buildout.cfg" {
142+
if err := base(ctx, path, BuildoutCfgComps); err != nil {
143+
return err
144+
}
137145
}
138146
return nil
139147
})
148+
149+
for k, v := range BuildoutCfgComps {
150+
if METADATAv, ok := MetadataComps[k]; !ok || METADATAv == "" {
151+
comps[[2]string{k, v}] = struct{}{}
152+
MetadataComps[k] = v
153+
}
154+
}
140155
var compList = maps.Keys(comps)
141156
if len(compList) == 0 {
142157
return nil, nil
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package buildout
2+
3+
import (
4+
"context"
5+
"github.com/murphysecurity/murphysec/infra/logctx"
6+
"go.uber.org/zap"
7+
"gopkg.in/ini.v1"
8+
"io"
9+
"net/http"
10+
"os"
11+
"path/filepath"
12+
"strings"
13+
)
14+
15+
func base(ctx context.Context, path string, result map[string]string) error {
16+
var log = logctx.Use(ctx).Sugar()
17+
e := findVersionsFile(ctx, path, result)
18+
if e == nil {
19+
return nil
20+
}
21+
pattern := filepath.Join(filepath.Dir(path), "*.cfg")
22+
files, err := filepath.Glob(pattern)
23+
if err != nil {
24+
log.Error("glob failed", zap.Error(e))
25+
return err
26+
}
27+
for _, j := range files {
28+
if j == "buildout.cfg" {
29+
continue
30+
}
31+
if err := NoCurrentDirectoryCfg(ctx, filepath.Dir(j), j, result); err != nil {
32+
return err
33+
}
34+
}
35+
return nil
36+
}
37+
func NoCurrentDirectoryCfg(ctx context.Context, NowPath string, path string, result map[string]string) error {
38+
var log = logctx.Use(ctx).Sugar()
39+
var e error
40+
var extends = ""
41+
if path == "" {
42+
return nil
43+
}
44+
if filepath.Dir(path) == NowPath {
45+
return nil
46+
}
47+
if strings.Contains(path, "http") {
48+
resp, err := http.Get(path)
49+
if err != nil {
50+
log.Error("http get failed", zap.Error(err))
51+
return err
52+
}
53+
defer resp.Body.Close()
54+
by, err := io.ReadAll(resp.Body)
55+
if err != nil {
56+
log.Error("read body failed", zap.Error(err))
57+
return err
58+
}
59+
extends, e = parseBuildoutBytes(ctx, by, result)
60+
if e != nil {
61+
return e
62+
}
63+
} else {
64+
// 如果不是远程连接 则尝试打开读取
65+
extends, e = parseBuildoutCfgFile(ctx, path, result)
66+
if e != nil {
67+
return e
68+
}
69+
}
70+
if extends != "" {
71+
log.Debug("find extends", zap.String("path", extends))
72+
return findVersionsFile(ctx, extends, result)
73+
}
74+
return NoCurrentDirectoryCfg(ctx, NowPath, extends, result)
75+
}
76+
func findVersionsFile(ctx context.Context, path string, result map[string]string) error {
77+
var log = logctx.Use(ctx).Sugar()
78+
var extends string
79+
var e error
80+
// 如果事远程链接 则读取
81+
if strings.Contains(path, "http") {
82+
resp, err := http.Get(path)
83+
if err != nil {
84+
log.Error("http get failed", zap.Error(err))
85+
return err
86+
}
87+
defer resp.Body.Close()
88+
by, err := io.ReadAll(resp.Body)
89+
if err != nil {
90+
log.Error("read body failed", zap.Error(err))
91+
return err
92+
}
93+
extends, e = parseBuildoutBytes(ctx, by, result)
94+
if e != nil {
95+
return e
96+
}
97+
} else {
98+
// 如果不是远程连接 则尝试打开读取
99+
extends, e = parseBuildoutCfgFile(ctx, path, result)
100+
if e != nil {
101+
return e
102+
}
103+
}
104+
if extends != "" {
105+
log.Debug("find extends", zap.String("path", extends))
106+
return findVersionsFile(ctx, extends, result)
107+
}
108+
109+
return nil
110+
}
111+
func parseBuildoutBytes(ctx context.Context, by []byte, result map[string]string) (string, error) {
112+
var log = logctx.Use(ctx).Sugar()
113+
cfg, err := ini.Load(by)
114+
if err != nil {
115+
log.Error("Fail to read file: ", zap.Error(err))
116+
return "", err
117+
}
118+
for _, section := range cfg.Sections() {
119+
if section.Name() == "version" || section.Name() == "dependencies" || section.Name() == "versions" {
120+
for _, key := range section.Keys() {
121+
if key.Name() != "" && key.Value() != "" {
122+
result[key.Name()] = key.Value()
123+
}
124+
}
125+
}
126+
}
127+
extends := cfg.Section("buildout").Key("extends").String()
128+
return extends, nil
129+
}
130+
func parseBuildoutCfgFile(ctx context.Context, path string, result map[string]string) (string, error) {
131+
var log = logctx.Use(ctx).Sugar()
132+
by, err := os.ReadFile(path)
133+
if err != nil {
134+
log.Error("read file failed", zap.Error(err))
135+
return "", err
136+
}
137+
cfg, err := ini.Load(by)
138+
if err != nil {
139+
log.Error("Fail to read file: ", zap.Error(err))
140+
return "", err
141+
}
142+
for _, section := range cfg.Sections() {
143+
if section.Name() == "version" || section.Name() == "dependencies" || section.Name() == "versions" {
144+
for _, key := range section.Keys() {
145+
if key.Name() != "" && key.Value() != "" {
146+
result[key.Name()] = key.Value()
147+
}
148+
}
149+
}
150+
}
151+
extends := cfg.Section("buildout").Key("extends").String()
152+
if extends == "" {
153+
return "", nil
154+
}
155+
if filepath.IsAbs(extends) {
156+
return extends, nil
157+
} else {
158+
return filepath.Join(filepath.Dir(path), extends), nil
159+
}
160+
}

0 commit comments

Comments
 (0)