Update 0.4.4 beta

Author: Ymjie

infoscan/api/api.go

@@ -4,13 +4,8 @@ import (
 	"GScan/infoscan/config"
 	"GScan/infoscan/dao"
 	"GScan/infoscan/service/Crawler"
-	"GScan/pkg"
-	"bufio"
 	"context"
 	"fmt"
-	"log"
-	"os"
-	"path/filepath"
 	"time"
 )
 
@@ -25,30 +20,12 @@ func NewApi(db dao.IDAO, config *config.Config) *Api {
 		config: config,
 	}
 }
-func (a *Api) StartCrawlerJob() (name string, id uint) {
+func (a *Api) StartCrawlerJob(urls []string) (name string, id uint) {
 	//pproff CPU占用监控
 	//pproff, _ := os.OpenFile(filepath.Join(Config.ResultPath, fmt.Sprintf("%s.cpu.pprof", time.Now().Format("2006-01-02 15-04-05"))), os.O_CREATE|os.O_RDWR, 0644)
 	//defer pproff.Close()
 	//pprof.StartCPUProfile(pproff)
 	//defer pprof.StopCPUProfile()
-	// url 列表读取
-	f, err := os.Open("url.txt")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer f.Close()
-	var urls []string
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		urls = append(urls, scanner.Text())
-	}
-	if err := scanner.Err(); err != nil {
-		log.Fatal(err)
-	}
-	fmt.Printf("扫描%d个URL.(Y/N):", len(urls))
-	if !pkg.AskForConfirmation() {
-		return "", 0
-	}
 	//启动任务
 	crawlerJob := Crawler.NewCrawlerJob(a.config, a.db, fmt.Sprintf("%s扫描%d个URL", time.Now().Format("2006-01-02 15-04-05"), len(urls)), urls)
 	ctx, _ := context.WithCancel(context.Background())
@@ -57,7 +34,8 @@ func (a *Api) StartCrawlerJob() (name string, id uint) {
 }
 
 func (a *Api) Out2Excel(jobID uint, filename string) {
-	Crawler.Out2Excel(jobID, a.db, filepath.Join(a.config.ResultPath, filename))
+	//Crawler.Out2Excel(jobID, a.db, filepath.Join(a.config.ResultPath, filename))
+	Crawler.Out2Excel(jobID, a.db, filename)
 }
 
 func (a *Api) GetJobs() {

infoscan/cmd/main.go

@@ -6,6 +6,8 @@ import (
 	"GScan/infoscan/dao/sqlite"
 	"GScan/pkg"
 	"GScan/pkg/logger"
+	"bufio"
+	"flag"
 	"fmt"
 	"log"
 	"os"
@@ -15,18 +17,33 @@ import (
 )
 
 var a *api.Api
+var Config *config.Config
+var urlf = flag.String("u", "url.txt", "url text path")
+var outfile = flag.String("o", "outfile.xlsx", "outfile excel path")
 
 func main() {
 	if len(os.Args) >= 2 {
 		switch os.Args[1] {
+		case "run":
+			//infoscan run -u url.txt -o name.xlsx
+			if err := flag.CommandLine.Parse(os.Args[2:]); err != nil {
+				log.Fatal(err)
+			}
+			urls, err := geturl(*urlf)
+			if err != nil {
+				log.Fatal(err)
+			}
+			_, jobID := a.StartCrawlerJob(urls)
+			a.Out2Excel(jobID, *outfile)
+			fmt.Print("Complete!")
 		case "ls":
 			a.GetJobs()
 		case "export":
 			atoi, err := strconv.Atoi(os.Args[2])
 			if err != nil {
 				log.Fatal(err)
 			}
-			a.Out2Excel(uint(atoi), fmt.Sprintf("%s-ExportJobID-%d.xlsx", time.Now().Format("2006-01-02 15-04-05"), atoi))
+			a.Out2Excel(uint(atoi), filepath.Join(Config.ResultPath, fmt.Sprintf("%s-ExportJobID-%d.xlsx", time.Now().Format("2006-01-02 15-04-05"), atoi)))
 		default:
 			fmt.Print(`InfoScan
 
@@ -36,12 +53,20 @@ Usage:
    infoscan.exe export <JobID>   #导出任务结果`)
 		}
 	} else {
-		jobname, jobID := a.StartCrawlerJob()
+		urls, err := geturl("url.txt")
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("扫描%d个URL.(Y/N):", len(urls))
+		if !pkg.AskForConfirmation() {
+			return
+		}
+		jobname, jobID := a.StartCrawlerJob(urls)
 		if jobID != 0 {
 			fmt.Printf("JobID:%d Name:%s 运行完成\n", jobID, jobname)
 			fmt.Printf("是否导出(Y/N):")
 			if pkg.AskForConfirmation() {
-				a.Out2Excel(jobID, jobname+".xlsx")
+				a.Out2Excel(jobID, filepath.Join(Config.ResultPath, jobname+".xlsx"))
 			}
 		}
 	}
@@ -54,7 +79,7 @@ func banner() {
 ██ ██  ██ ██ ██      ██    ██      ██ ██      ██   ██ ██  ██ ██ 
 ██ ██   ████ ██       ██████  ███████  ██████ ██   ██ ██   ████ 
 ---------------------------------------------------------------
-Version: 0.4.2beta
+Version: 0.4.4beta
 Email:   i@vshex.com
 Github:  https://github.com/Ymjie/GScan
 ---------------------------------------------------------------`)
@@ -67,7 +92,7 @@ func init() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	Config := c
+	Config = c
 	os.Mkdir(Config.ResultPath, 0644)
 	os.Mkdir(Config.LogPath, 0644)
 	//设置日志
@@ -80,3 +105,20 @@ func init() {
 	//api 初始化
 	a = api.NewApi(DB, Config)
 }
+func geturl(urlpath string) ([]string, error) {
+	// url 列表读取
+	f, err := os.Open(urlpath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var urls []string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		urls = append(urls, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return urls, nil
+}

infoscan/cmd/whitelist.txt

@@ -43,4 +43,5 @@ baike.com
 dzwww.com
 e23.cn
 dzng.com
-ihwrm.com
+ihwrm.com
+ql1d.com