Merge branch 'master' of github.com:akkuman/rotateproxy

Author: akkuman

.github/workflows/release.yml

@@ -7,21 +7,22 @@ on:
 
 jobs:
   build-rotateproxy:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Setup Golang
-        uses: actions/setup-go@v2
+      - name: Set up Go
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.17
+          go-version: 1.21
+          cache: true
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v4
+        uses: goreleaser/goreleaser-action@v5
         with:
+          distribution: goreleaser
           version: latest
           args: release --clean
-          workdir: cmd/rotateproxy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,6 +1,9 @@
 *.exe
+*.exe~
 .history
 .vscode
 *.zip
-cmd/rotateproxy/db.db
-build
+build
+*.ps1
+*.db
+cmd/rotateproxy/db.db*

.goreleaser.yml

@@ -1,19 +1,34 @@
+before:
+  hooks:
+    - go mod tidy
 builds:
-  - skip: true
+- env: [CGO_ENABLED=0]
+  goos:
+    - linux
+    - windows
+    - darwin
+    - freebsd
+  goarch:
+    - amd64
+    - arm64
+  dir: .
+  main: ./cmd/rotateproxy
+  flags:
+    # trims path
+    - -trimpath
+  ldflags:
+    # use commit date instead of current date as main.date
+    # only needed if you actually use those things in your main package, otherwise can be ignored.
+    - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }}
 checksum:
   name_template: 'checksums.txt'
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
 release:
-  # Same as for github
-  # Note: it can only be one: either github, gitlab or gitea
-  github:
-    owner: akkuman
-    name: rotateproxy
-
-  draft: false
-
-  # You can add extra pre-existing files to the release.
-  # The filename on the release will be the last part of the path (base). If
-  # another file with the same name exists, the latest one found will be used.
-  # Defaults to empty.
-  extra_files:
-    - glob: ./build/*
+  name_template: 'v{{ .Version }}'
+archives:
+  - format: zip

README.md

@@ -1,6 +1,6 @@
 # rotateproxy
 
-利用fofa搜索socks5开放代理进行代理池轮切的工具
+利用 fofa 搜索 socks5 开放代理进行代理池轮切的工具
 
 ## 特性
 
@@ -10,8 +10,12 @@
 ## 帮助
 
 ```shell
-> .\rotateproxy.exe -h
+.\rotateproxy.exe -h
 Usage of rotateproxy.exe:
+  -check string
+        check url (default "https://www.google.com")
+  -checkWords string
+        words in check url (default "Copyright The Closure Library Authors")
   -email string
         email address
   -l string
@@ -20,16 +24,30 @@ Usage of rotateproxy.exe:
         the page count you want to crawl (default 5)
   -pass string
         authentication password
+  -proxy string
+        proxy
   -region int
         0: all 1: cannot bypass gfw 2: bypass gfw
   -rule string
         search rule (default "protocol==\"socks5\" && \"Version:5 Method:No Authentication(0x00)\" && after=\"2025-03-01\" && country=\"CN\"")
+  -strategy int
+        0: random, 1: Select the one with the shortest timeout (default 1)
   -token string
         token
   -user string
         authentication username
 ```
 
+### 参数说明
+
+- `-check`，`-checkWords`: 如果你希望你找出来的代理能够访问某个特定网站，可以使用 `-check` 和 `-checkWords`，checkWords 为一串字符，用来保证确定是访问的该网站，比如 `-check "https://www.google.com" -checkWords "Copyright The Closure Library Authors"`，目的就是找出来的代理能够访问 [https://www.google.com](https://www.google.com)，并且返回内容中含有 `Copyright The Closure Library Authors`
+- `-email`，`-token`: fofa 的邮箱和 token
+- `-l`: 本服务的 socks5 监听地址
+- `-page`: fofa 结果默认爬取多少页（每页固定爬取 100 条）
+- `-user`，`-pass`: 本服务的 socks5 用户名密码认证
+- `-proxy`: 如果提供，将使用该 proxy 来访问 fofa 获取代理地址
+- `-region`: 用来筛选是否能访问谷歌
+
 ## 安装
 
 ```shell
@@ -43,7 +61,7 @@ curl -L -o rotateproxy 'https://github.com/akkuman/rotateproxy/releases/latest/d
 chmod +x ./rotateproxy
 ```
 
-### 安装为linux服务（感谢 [@Rvn0xsy](https://github.com/Rvn0xsy) 提供 [PR](https://github.com/akkuman/rotateproxy/pull/4)）
+### 安装为 linux 服务（感谢 [@Rvn0xsy](https://github.com/Rvn0xsy) 提供 [PR](https://github.com/akkuman/rotateproxy/pull/4)）
 
 1. 下载相关文件
 
@@ -54,6 +72,7 @@ curl -L -o /usr/lib/systemd/system/rotateproxy.service 'https://raw.githubuserco
 ```
 
 2. 查看 [rotateproxy.service.example](./rotateproxy.service.example) 文件示例，将 `/usr/lib/systemd/system/rotateproxy.service` 文件中的 `ExecStart` 的命令替换为你自己的命令
+
 3. 启动服务
 
 ```shell
@@ -70,17 +89,15 @@ systemctl enable rotateproxy.service
 4. 使用
 
 ```bash
-https_proxy=socks5://{ip}:{port} curl https://example.com
+curl --proxy "socks5://127.0.0.1:8899" https://example.com
 ```
 
 ## 效果展示
 
 ![](./pics/curl-run.jpg)
 
-## mac 交叉编译
+## 开发
 
-```bash
-brew install FiloSottile/musl-cross/musl-cross
-cd cmd/rotateproxy
-CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags "-linkmode external -extldflags -static"
-```
+### 注意
+
+1. 该项目为了交叉编译方便，采用的 cgo-free 的 sqlite3 库，该库默认可能开启了 `SQLITE_FCNTL_PERSIST_WAL`，导致就算 Close 了数据库连接依旧不会删除 wal 文件，所以可能你会在 .db 旁边看到 -shm 和 -wal 文件，如果需要共享该数据库，请将这三个文件一并打包发送

check.go

@@ -1,11 +1,12 @@
 package rotateproxy
 
 import (
+	"context"
 	"crypto/tls"
-	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/tidwall/gjson"
@@ -36,7 +37,8 @@ func CheckProxyAlive(proxyURL string) (respBody string, timeout int64, avail boo
 			TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
 			DisableKeepAlives: true,
 		},
-		Timeout: 20 * time.Second,
+		// shorter timeout for better proxies
+		Timeout: 5 * time.Second,
 	}
 	startTime := time.Now()
 	resp, err := httpclient.Get("https://whois.pconline.com.cn/ipJson.jsp?json=true&ip=")
@@ -45,7 +47,7 @@ func CheckProxyAlive(proxyURL string) (respBody string, timeout int64, avail boo
 	}
 	defer resp.Body.Close()
 	timeout = int64(time.Since(startTime))
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", 0, false
 	}
@@ -55,8 +57,8 @@ func CheckProxyAlive(proxyURL string) (respBody string, timeout int64, avail boo
 	return string(body), timeout, true
 }
 
-func CheckProxyWithCheckURL(proxyURL string, checkURL string) (timeout int64, avail bool) {
-	fmt.Printf("check %s： %s\n", proxyURL, checkURL)
+func CheckProxyWithCheckURL(proxyURL string, checkURL string, checkURLwords string) (timeout int64, avail bool) {
+	// InfoLog(Notice("check %s： %s", proxyURL, checkURL))
 	proxy, _ := url.Parse(proxyURL)
 	httpclient := &http.Client{
 		Transport: &http.Transport{
@@ -73,46 +75,58 @@ func CheckProxyWithCheckURL(proxyURL string, checkURL string) (timeout int64, av
 	}
 	defer resp.Body.Close()
 	timeout = int64(time.Since(startTime))
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		return 0, false
+	}
 
 	// TODO: support regex
 	if resp.StatusCode != 200 {
 		return 0, false
 	}
 
+	if !strings.Contains(string(body), checkURLwords) {
+		return 0, false
+	}
+
 	return timeout, true
 }
 
-func StartCheckProxyAlive(checkURL string) {
+func StartCheckProxyAlive(ctx context.Context, checkURL string, checkURLwords string) {
 	go func() {
 		ticker := time.NewTicker(120 * time.Second)
+		defer ticker.Stop()
 		for {
 			select {
 			case <-crawlDone:
-				fmt.Println("Checking")
-				checkAlive(checkURL)
-				fmt.Println("Check done")
+				InfoLog(Noticeln("Checkings"))
+				checkAlive(checkURL, checkURLwords)
+				InfoLog(Noticeln("Check done"))
 			case <-ticker.C:
-				checkAlive(checkURL)
+				checkAlive(checkURL, checkURLwords)
+			case <- ctx.Done():
+				return
 			}
 		}
 	}()
 }
 
-func checkAlive(checkURL string) {
+func checkAlive(checkURL string, checkURLwords string) {
 	proxies, err := QueryProxyURL()
 	if err != nil {
-		fmt.Printf("[!] query db error: %v\n", err)
+		ErrorLog(Warn("[!] query db error: %v", err))
 	}
 	for i := range proxies {
 		proxy := proxies[i]
 		go func() {
 			respBody, timeout, avail := CheckProxyAlive(proxy.URL)
 			if avail {
 				if checkURL != "" {
-					timeout, avail = CheckProxyWithCheckURL(proxy.URL, checkURL)
+					timeout, avail = CheckProxyWithCheckURL(proxy.URL, checkURL, checkURLwords)
 				}
 				if avail {
-					fmt.Printf("%v 可用\n", proxy.URL)
+					InfoLog(Notice("%v 可用", proxy.URL))
 					SetProxyURLAvail(proxy.URL, timeout, CanBypassGFW(respBody))
 					return
 				}