Archiving improve

1. Fixed archiving multiple URI only one result.
2. Tor onion got a priority.

Author: web-flow

README.md

@@ -60,7 +60,7 @@ func main() {
 
 ## FAQ
 
-### Archive.today is unavailable?
+### archive.today is unavailable?
 
 Archive.today may have enforced a strictly CAPTCHA policy, causing an exception to the request.
 

go.mod

@@ -5,5 +5,6 @@ go 1.15
 require (
 	github.com/PuerkitoBio/goquery v1.6.1
 	github.com/cretz/bine v0.1.0
-	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616
 )

go.sum

@@ -4,6 +4,15 @@ github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5z
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/cretz/bine v0.1.0 h1:1/fvhLE+fk0bPzjdO5Ci+0ComYxEMuB1JhM4X5skT3g=
 github.com/cretz/bine v0.1.0/go.mod h1:6PF6fWAvYtwjRGkAuDEJeWNOv3a2hUouSP/yRYXmvHw=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616 h1:wZ5HtpmZAVUq0Im5Sm92ycJrTeLJk5lB/Kvh55Rd+Ps=
+github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -12,3 +21,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/http.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -26,7 +25,6 @@ type Archiver struct {
 
 	final    string
 	submitid string
-	isTor    bool
 
 	httpClient *http.Client
 	torClient  *http.Client
@@ -69,17 +67,17 @@ func (wbrc *Archiver) fetch(s string, ch chan<- string) {
 			break
 		}
 	}
-	r(domains)
 
-	if baseuri == nil || wbrc.submitid == "" {
-		// Try request over Tor hidden service.
-		if wbrc.torClient == nil {
-			ch <- fmt.Sprint("Tor network unreachable.")
-			return
-		}
+	// Try request over Tor hidden service.
+	if wbrc.torClient != nil {
 		wbrc.httpClient = wbrc.torClient
 
 		r([]string{onion})
+	}
+	defer wbrc.clear()
+
+	if baseuri == nil || wbrc.submitid == "" {
+		r(domains)
 		if baseuri == nil || wbrc.submitid == "" {
 			ch <- fmt.Sprint("archive.today is unavailable.")
 			return
@@ -190,9 +188,8 @@ func (wbrc *Archiver) getSubmitID(url string) (string, error) {
 	return id, nil
 }
 
-func isURL(str string) bool {
-	re := regexp.MustCompile(`https?://?[-a-zA-Z0-9@:%._\+~#=]{1,255}\.[a-z]{0,63}\b(?:[-a-zA-Z0-9@:%_\+.~#?&//=]*)`)
-	match := re.FindAllString(str, -1)
-
-	return len(match) >= 1
+func (wbrc *Archiver) clear() {
+	baseuri = nil
+	wbrc.final = ""
+	wbrc.submitid = ""
 }

pkg/is.go

@@ -3,33 +3,42 @@ package is
 import (
 	"log"
 	"strings"
+	"sync"
+
+	"github.com/wabarc/helper"
 )
 
 // Wayback is the handle of saving webpages to archive.is
 func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) {
 	collect := make(map[string]string)
 	for _, link := range links {
-		if !isURL(link) {
+		if !helper.IsURL(link) {
 			log.Print(link + " is invalid url.")
 			continue
 		}
 		collect[link] = link
 	}
 
-	ch := make(chan string, len(collect))
-	defer close(ch)
-
 	if client, tor, err := wbrc.newTorClient(); err != nil {
 		log.Println(err)
 	} else {
 		wbrc.torClient = client
 		defer tor.Close()
 	}
 
+	ch := make(chan string, len(collect))
+	defer close(ch)
+
+	var wg sync.WaitGroup
 	for link := range collect {
-		go wbrc.fetch(link, ch)
-		collect[link] = strings.Replace(<-ch, onion, "archive.today", 1)
+		wg.Add(1)
+		go func(link string, ch chan string) {
+			wbrc.fetch(link, ch)
+			collect[link] = strings.Replace(<-ch, onion, "archive.today", 1)
+			wg.Done()
+		}(link, ch)
 	}
+	wg.Wait()
 
 	return collect, nil
 }