Skip to content

Commit 72daf24

Browse files
bufsnakebufsnake
committed
add passive scan and can -b delete junk data
1 parent bea4d09 commit 72daf24

File tree

12 files changed

+989
-219
lines changed

12 files changed

+989
-219
lines changed

‎README.md‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,34 +18,50 @@ go build -v
1818
## 使用
1919

2020
```bash
21-
└> ./blueming
2221
Usage of ./blueming:
23-
-es string
24-
dirscan filter status(200,206,301,302,400,401,403,404,405,500,501,502,503,504,600,etc.) (default "404")
22+
-b filter output data
23+
-crt string
24+
listen cert (default "ca.crt")
2525
-f string
2626
set url file
2727
-i string
28-
set wordlist index(ex: test.php)
28+
set wordlist index(exp: test.php)
29+
-key string
30+
listen key (default "ca.key")
2931
-l string
3032
set log level(trace,debug,info,warn,fatal) (default "debug")
33+
-listen string
34+
listen to scan dir (default "127.0.0.1:9099")
3135
-p string
32-
set download proxy
36+
set proxy, support http proxy(exp: http://localhost:8080)
3337
-s int
3438
set timeout (default 10)
3539
-t int
36-
set thread (default 10)
40+
set thread (default 100)
3741
-u string
3842
set url
43+
-v int
44+
log level
3945
-w string
4046
set wordlist
4147
```
4248

49+
> ./blueming -b 可删除output下的垃圾数据(必须使用)
50+
4351
## TODO
4452

4553
> 基本满足以下要求即可
4654
55+
- [ ] 常见文件泄露扫描 .git .hg .idea .DS_Store ...
56+
- [x] 开启被动扫描模式，配合httpx自动进行目录扫描(二级、三级、四级...)
4757
- [x] 通过URL自动生成文件名
4858
- [x] 根据后缀名将URL定义为对应的文件格式，如zip、tar.gz等
4959
- [x] 自动下载备份文件，并进行重命名
5060
- [x] 能够自定义字典
5161
- [x] 优化内存占用
62+
- [x] filter.sh 移至程序内部
63+
- [x] 目录扫描部分添加 页面相似度比较，每个新产生的都会与前面所有的请求进行比较一次(耗时)
64+
- 比较时，各网站相互独立，采用协程的方式
65+
- [x] 采用 GET 请求，查看文件过大时的response
66+
- 文件过大导致的超时 则获取 header，比较历史记录中的length
67+
- 正常情况，比较body

‎cmd/blueming/filter.sh‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎cmd/blueming/main.go‎

Lines changed: 140 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,90 @@ import (
66
"github.com/bufsnake/blueming/config"
77
"github.com/bufsnake/blueming/internal/core"
88
"github.com/bufsnake/blueming/pkg/log"
9+
"golang.org/x/text/encoding/simplifiedchinese"
910
"io/ioutil"
11+
"math"
1012
"net/url"
1113
"os"
14+
"os/exec"
15+
"runtime"
1216
"strings"
17+
"sync"
18+
"syscall"
1319
"time"
1420
)
1521

1622
func main() {
1723
conf := config.Config{}
18-
flag.IntVar(&conf.Thread, "t", 10, "set thread")
24+
flag.IntVar(&conf.Thread, "t", 100, "set thread")
1925
flag.IntVar(&conf.Timeout, "s", 10, "set timeout")
2026
flag.StringVar(&conf.Url, "u", "", "set url")
2127
flag.StringVar(&conf.Urlfile, "f", "", "set url file")
2228
flag.StringVar(&conf.Loglevel, "l", log.DEBUG, "set log level(trace,debug,info,warn,fatal)")
2329
flag.StringVar(&conf.Wordlist, "w", "", "set wordlist")
24-
flag.StringVar(&conf.Index, "i", "", "set wordlist index(ex: test.php)")
25-
flag.StringVar(&conf.Proxy, "p", "", "set download proxy")
26-
flag.StringVar(&conf.ExcludeStatus, "es", "404", "dirscan filter status(200,206,301,302,307,400,401,402,403,404,405,406,424,500,501,502,503,504,600,etc.)")
27-
flag.StringVar(&conf.ResultFile, "rf", "", "parse result file")
30+
flag.StringVar(&conf.Index, "i", "", "set wordlist index(exp: test.php)")
31+
flag.StringVar(&conf.Proxy, "p", "", "set proxy, support http proxy(exp: http://localhost:8080)")
32+
flag.StringVar(&conf.Listen, "listen", "127.0.0.1:9099", "listen to scan dir")
33+
flag.StringVar(&conf.URLStrs, "urls", "", "set url file")
34+
flag.StringVar(&conf.Cert, "crt", "ca.crt", "listen cert")
35+
flag.StringVar(&conf.Key, "key", "ca.key", "listen key")
36+
flag.BoolVar(&conf.FilterOutput, "b", false, "filter output data")
37+
// 暂不考虑
38+
//flag.StringVar(&conf.ResultFile, "rf", "", "parse result file")
2839
flag.Parse()
40+
// 开启多核模式
41+
runtime.GOMAXPROCS(runtime.NumCPU() * 3 / 4)
42+
// 关闭 GIN Debug模式
43+
// 设置工具可打开的文件描述符
44+
var rLimit syscall.Rlimit
45+
rLimit.Max = 999999
46+
rLimit.Cur = 999999
47+
if runtime.GOOS == "darwin" {
48+
rLimit.Cur = 10240
49+
}
50+
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
51+
if err != nil {
52+
fmt.Println(err)
53+
os.Exit(1)
54+
}
55+
_ = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
2956
log.SetLevel(conf.Loglevel)
57+
if conf.FilterOutput {
58+
// 获取 output 下的所有文件 不包含文件夹
59+
allfiles, _ := ioutil.ReadDir("./output")
60+
for _,f := range allfiles {
61+
if !f.IsDir() {
62+
if f.Size() <= 1048576 {
63+
err = os.Remove("./output/" + f.Name())
64+
if err != nil {
65+
log.Fatal(err)
66+
}
67+
}
68+
}
69+
}
70+
71+
wait := sync.WaitGroup{}
72+
files, _ := ioutil.ReadDir("./output")
73+
fmt.Println("current exist", len(files), "files")
74+
go func() {
75+
for {
76+
fmt.Printf("\r%.2f%%", math.Trunc(((increase/float64(len(files)))*100)*1e2)*1e-2)
77+
time.Sleep(1 * time.Second / 10)
78+
}
79+
}()
80+
for _, f := range files {
81+
if !f.IsDir() {
82+
wait.Add(1)
83+
go filter(&wait, strings.ReplaceAll("./output/"+f.Name(), " ", ` `), float64(len(files)))
84+
} else {
85+
increaseAdd()
86+
fmt.Printf("\r%.2f%%", math.Trunc(((increase/float64(len(files)))*100)*1e2)*1e-2)
87+
}
88+
}
89+
wait.Wait()
90+
// function filter { if [[ $(file $1 | grep $1": data") == "" && $(file $1 | grep "image data") == "" && $(file $1 | grep "HTML") == "" && $(file $1 | grep "empty") == "" && $(file $1 | grep "JSON") == "" && $(file $1 | grep "text") == "" ]]; then file $1; else rm -rf $1; fi } && filter logs/data.tar.gz
91+
os.Exit(1)
92+
}
3093
urls := []string{}
3194
if conf.Url != "" {
3295
urls = append(urls, conf.Url)
@@ -88,12 +151,43 @@ func main() {
88151
}
89152
}
90153
return
154+
} else if conf.Listen != "" {
155+
if conf.Wordlist == "" {
156+
log.Fatal("If passive scanning is started, a dictionary must be specified")
157+
}
158+
if conf.URLStrs == "" {
159+
log.Fatal("urls must be specified")
160+
}
161+
passive := core.NewPassive(conf)
162+
err = passive.Start()
163+
if err != nil {
164+
log.Fatal(err)
165+
}
91166
} else {
92167
flag.Usage()
93168
return
94169
}
170+
// 判断 output 文件夹是否存在
171+
if !exists("./output") {
172+
log.Info("create output file path")
173+
err := os.Mkdir("./output/", os.ModePerm)
174+
if err != nil {
175+
log.Warn("create output file path error", err)
176+
os.Exit(1)
177+
}
178+
}
179+
// 创建 Log 文件夹
180+
if !exists("./logs") {
181+
log.Info("create logs file path")
182+
err := os.Mkdir("./logs/", os.ModePerm)
183+
if err != nil {
184+
log.Warn("create logs file path error", err)
185+
os.Exit(1)
186+
}
187+
}
188+
95189
log.Info(len(urls), "个URL,", conf.Thread, "线程,", conf.Timeout, "超时")
96-
config.LogFileName = "Log-" + time.Now().Format("2006-01-02 15:04:05")
190+
config.LogFileName = "./logs/Log-" + time.Now().Format("2006-01-02 15:04:05")
97191
create, err := os.Create(config.LogFileName)
98192
if err != nil {
99193
log.Warn(err)
@@ -105,3 +199,43 @@ func main() {
105199
newCore := core.NewCore(urls, conf)
106200
newCore.Core()
107201
}
202+
203+
func exists(path string) bool {
204+
_, err := os.Stat(path) //os.Stat获取文件信息
205+
if err != nil {
206+
if os.IsExist(err) {
207+
return true
208+
}
209+
return false
210+
}
211+
return true
212+
}
213+
214+
var increase float64 = 0
215+
var inc_l sync.Mutex
216+
217+
func increaseAdd() {
218+
inc_l.Lock()
219+
defer inc_l.Unlock()
220+
increase++
221+
}
222+
223+
func filter(wait *sync.WaitGroup, filename string, totalcount float64) {
224+
defer wait.Done()
225+
bin := []string{"-c", "function filter { if [[ $(file $1 | grep $1\": data\") == \"\" && $(file $1 | grep \"image data\") == \"\" && $(file $1 | grep \"HTML\") == \"\" && $(file $1 | grep \"empty\") == \"\" && $(file $1 | grep \"JSON\") == \"\" && $(file $1 | grep \"text\") == \"\" ]]; then file $1; else rm -rf $1; fi } && filter '" + filename + "'"}
226+
// 其他的shell环境太烦了
227+
run := exec.Command("/bin/zsh", bin...)
228+
output, err := run.Output()
229+
if err != nil {
230+
log.Fatal(err)
231+
}
232+
output, err = simplifiedchinese.GB18030.NewDecoder().Bytes(output)
233+
if err != nil {
234+
log.Fatal(err)
235+
}
236+
if len(output) != 0 {
237+
fmt.Print("\r" + string(output))
238+
}
239+
increaseAdd()
240+
fmt.Printf("\r%.2f%%", math.Trunc(((increase/totalcount)*100)*1e2)*1e-2)
241+
}

‎config/main.go‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,28 @@
11
package config
22

33
type Config struct {
4-
Thread int
5-
Timeout int
6-
Url string
7-
Urlfile string
8-
Loglevel string
9-
Wordlist string
10-
Index string
11-
Proxy string
12-
ExcludeStatus string
13-
ResultFile string
4+
Thread int
5+
Timeout int
6+
Url string
7+
Urlfile string
8+
Loglevel string
9+
Wordlist string
10+
Index string
11+
Proxy string
12+
ResultFile string
13+
FilterOutput bool // 过滤 output 文件夹中的垃圾数据
14+
Listen string
15+
URLStrs string
16+
Cert string
17+
Key string
1418
}
1519

1620
var LogFileName string
21+
22+
type HTTPStatus struct {
23+
URL string
24+
Status int
25+
ContentType string
26+
Size string
27+
Body string
28+
}

‎go.mod‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,10 @@ module github.com/bufsnake/blueming
22

33
go 1.14
44

5-
require github.com/logrusorgru/aurora v2.0.3+incompatible
5+
require (
6+
github.com/antlabs/strsim v0.0.2
7+
github.com/google/martian v2.1.0+incompatible
8+
github.com/logrusorgru/aurora v2.0.3+incompatible
9+
github.com/weppos/publicsuffix-go v0.15.0
10+
golang.org/x/text v0.3.0
11+
)

‎go.sum‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,20 @@
1+
github.com/antlabs/strsim v0.0.2 h1:R4qjokEegYTrw+fkcYj3/UndG9Cn136fH+fpw9TIz9k=
2+
github.com/antlabs/strsim v0.0.2/go.mod h1:95XAAF2dJK9IiZMc0Ue6H9t477/i6fvYoMoeey8sEnc=
3+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4+
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
5+
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
16
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
27
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
8+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
9+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
10+
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
11+
github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ=
12+
github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
13+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
14+
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
15+
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
16+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
17+
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
18+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
19+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
20+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0 commit comments

Comments
 (0)