Minor improvements

Author: web-flow

.gitignore

@@ -0,0 +1,15 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/

Makefile

@@ -180,4 +180,4 @@ fmt:
 
 test:
 	@echo "-> Running go test"
-	go test -v ./...
+	@CGO_ENABLED=1 go test -v -race -cover -coverprofile=coverage.out -covermode=atomic ./...

pkg/http.go

@@ -1,11 +1,9 @@
 package is
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -16,16 +14,13 @@ import (
 	"github.com/PuerkitoBio/goquery"
 )
 
-type Archiver struct {
-	Anyway string
-	Cookie string
+type IS struct {
+	wbrc *Archiver
 
-	DialContext         func(ctx context.Context, network, addr string) (net.Conn, error)
-	SkipTLSVerification bool
-
-	final    string
 	submitid string
+	final    string
 
+	baseuri    *url.URL
 	httpClient *http.Client
 	torClient  *http.Client
 }
@@ -37,7 +32,6 @@ var (
 	onion     = "archivecaslytosk.onion" // archiveiya74codqgiixo33q62qlrqtkgmcitqx5u2oeqnmn5bpcbiyd.onion
 	cookie    = ""
 	timeout   = 120 * time.Second
-	baseuri   *url.URL
 	domains   = []string{
 		"archive.today",
 		"archive.is",
@@ -49,59 +43,54 @@ var (
 	}
 )
 
-func (wbrc *Archiver) fetch(s string, ch chan<- string) {
-	wbrc.httpClient = &http.Client{
-		Timeout: timeout,
-	}
-
+func (is *IS) fetch(s string, ch chan<- string) {
 	// get valid domain and submitid
 	r := func(domains []string) {
 		for _, domain := range domains {
 			h := fmt.Sprintf("%v://%v", scheme, domain)
-			id, err := wbrc.getSubmitID(h)
+			id, err := is.getSubmitID(h)
 			if err != nil {
 				continue
 			}
-			baseuri, _ = url.Parse(h)
-			wbrc.submitid = id
+			is.baseuri, _ = url.Parse(h)
+			is.submitid = id
 			break
 		}
 	}
 
 	// Try request over Tor hidden service.
-	if wbrc.torClient != nil {
-		wbrc.httpClient = wbrc.torClient
+	if is.torClient != nil {
+		is.httpClient = is.torClient
 
 		r([]string{onion})
 	}
-	defer wbrc.clear()
 
-	if baseuri == nil || wbrc.submitid == "" {
+	if is.baseuri == nil || is.submitid == "" {
 		r(domains)
-		if baseuri == nil || wbrc.submitid == "" {
+		if is.baseuri == nil || is.submitid == "" {
 			ch <- fmt.Sprint("archive.today is unavailable.")
 			return
 		}
 	}
 
-	if wbrc.Anyway != "" {
-		anyway = wbrc.Anyway
+	if is.wbrc.Anyway != "" {
+		anyway = is.wbrc.Anyway
 	}
 	data := url.Values{
-		"submitid": {wbrc.submitid},
+		"submitid": {is.submitid},
 		"anyway":   {anyway},
 		"url":      {s},
 	}
-	uri := baseuri.String()
-	req, err := http.NewRequest("POST", baseuri.String()+"/submit/", strings.NewReader(data.Encode()))
+	uri := is.baseuri.String()
+	req, err := http.NewRequest("POST", is.baseuri.String()+"/submit/", strings.NewReader(data.Encode()))
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 	req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
 	req.Header.Add("User-Agent", userAgent)
 	req.Header.Add("Referer", uri)
 	req.Header.Add("Origin", uri)
-	req.Header.Add("Host", baseuri.Hostname())
-	req.Header.Add("Cookie", wbrc.getCookie())
-	resp, err := wbrc.httpClient.Do(req)
+	req.Header.Add("Host", is.baseuri.Hostname())
+	req.Header.Add("Cookie", is.getCookie())
+	resp, err := is.httpClient.Do(req)
 	if err != nil {
 		ch <- fmt.Sprint(err)
 		return
@@ -124,38 +113,38 @@ func (wbrc *Archiver) fetch(s string, ch chan<- string) {
 	// Redirect to final url if page saved.
 	final := resp.Request.URL.String()
 	if len(final) > 0 && strings.Contains(final, "/submit/") == false {
-		wbrc.final = final
+		is.final = final
 	}
 	loc := resp.Header.Get("location")
 	if len(loc) > 2 {
-		wbrc.final = loc
+		is.final = loc
 	}
 	// When use anyway parameter.
 	refresh := resp.Header.Get("refresh")
 	if len(refresh) > 0 {
 		r := strings.Split(refresh, ";url=")
 		if len(r) == 2 {
-			wbrc.final = r[1]
+			is.final = r[1]
 		}
 	}
 
-	ch <- wbrc.final
+	ch <- is.final
 }
 
-func (wbrc *Archiver) getCookie() string {
+func (is *IS) getCookie() string {
 	c := os.Getenv("ARCHIVE_COOKIE")
 	if c != "" {
-		wbrc.Cookie = c
+		is.wbrc.Cookie = c
 	}
 
-	if wbrc.Cookie != "" {
-		return wbrc.Cookie
+	if is.wbrc.Cookie != "" {
+		return is.wbrc.Cookie
 	} else {
 		return cookie
 	}
 }
 
-func (wbrc *Archiver) getSubmitID(url string) (string, error) {
+func (is *IS) getSubmitID(url string) (string, error) {
 	if strings.Contains(url, "http") == false {
 		return "", fmt.Errorf("missing protocol scheme")
 	}
@@ -164,8 +153,8 @@ func (wbrc *Archiver) getSubmitID(url string) (string, error) {
 	req, err := http.NewRequest("GET", url, r)
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 	req.Header.Add("User-Agent", userAgent)
-	req.Header.Add("Cookie", wbrc.getCookie())
-	resp, err := wbrc.httpClient.Do(req)
+	req.Header.Add("Cookie", is.getCookie())
+	resp, err := is.httpClient.Do(req)
 	if err != nil {
 		return "", err
 	}
@@ -187,9 +176,3 @@ func (wbrc *Archiver) getSubmitID(url string) (string, error) {
 
 	return id, nil
 }
-
-func (wbrc *Archiver) clear() {
-	baseuri = nil
-	wbrc.final = ""
-	wbrc.submitid = ""
-}

pkg/is.go

@@ -1,16 +1,28 @@
 package is
 
 import (
+	"context"
+	"fmt"
 	"log"
+	"net"
+	"net/http"
 	"strings"
 	"sync"
 
 	"github.com/wabarc/helper"
 )
 
+type Archiver struct {
+	Anyway string
+	Cookie string
+
+	DialContext         func(ctx context.Context, network, addr string) (net.Conn, error)
+	SkipTLSVerification bool
+}
+
 // Wayback is the handle of saving webpages to archive.is
 func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) {
-	collect := make(map[string]string)
+	collect, results := make(map[string]string), make(map[string]string)
 	for _, link := range links {
 		if !helper.IsURL(link) {
 			log.Print(link + " is invalid url.")
@@ -19,26 +31,40 @@ func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) {
 		collect[link] = link
 	}
 
-	if client, tor, err := wbrc.newTorClient(); err != nil {
+	torClient, tor, err := newTorClient()
+	if err != nil {
 		log.Println(err)
 	} else {
-		wbrc.torClient = client
 		defer tor.Close()
 	}
 
 	ch := make(chan string, len(collect))
 	defer close(ch)
 
+	var mu sync.Mutex
 	var wg sync.WaitGroup
 	for link := range collect {
 		wg.Add(1)
-		go func(link string, ch chan string) {
-			wbrc.fetch(link, ch)
-			collect[link] = strings.Replace(<-ch, onion, "archive.today", 1)
+		go func(link string) {
+			is := &IS{
+				wbrc:       wbrc,
+				httpClient: &http.Client{Timeout: timeout},
+				torClient:  torClient,
+				final:      "",
+				submitid:   "",
+			}
+			is.fetch(link, ch)
+			mu.Lock()
+			results[link] = strings.Replace(<-ch, onion, "archive.today", 1)
+			mu.Unlock()
 			wg.Done()
-		}(link, ch)
+		}(link)
 	}
 	wg.Wait()
 
-	return collect, nil
+	if len(results) == 0 {
+		return results, fmt.Errorf("No results")
+	}
+
+	return results, nil
 }

pkg/is_test.go

@@ -19,7 +19,7 @@ func TestWayback(t *testing.T) {
 	links = []string{"https://www.bbc.com/", "https://www.google.com/"}
 	got, _ = wbrc.Wayback(links)
 	if len(got) == 0 {
-		t.Errorf("got = %d; want not equal 0", len(got))
+		t.Errorf("got = %d; want greater than 0", len(got))
 	}
 
 	for orig, dest := range got {

pkg/tor.go

@@ -15,7 +15,7 @@ import (
 	// "golang.org/x/net/proxy"
 )
 
-func (arc *Archiver) newTorClient() (*http.Client, *tor.Tor, error) {
+func newTorClient() (*http.Client, *tor.Tor, error) {
 	// Lookup tor executable file
 	if _, err := exec.LookPath("tor"); err != nil {
 		return nil, nil, fmt.Errorf("%w", err)