Fix UA robots check

Author: StJudeWasHere

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/gorilla/sessions v1.4.0
 	github.com/gorilla/websocket v1.5.3
 	github.com/microcosm-cc/bluemonday v1.0.27
+	github.com/mileusna/useragent v1.3.5
 	github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4
 	github.com/slyrz/warc v0.0.0-20150806225202-a50edd19b690
 	github.com/spf13/viper v1.20.1

go.sum

@@ -78,6 +78,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
 github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
+github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws=
+github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=

internal/crawler/basic_client.go

@@ -6,6 +6,8 @@ import (
 	"net/http/httptrace"
 	"net/url"
 	"time"
+
+	"github.com/mileusna/useragent"
 )
 
 type HTTPRequester interface {
@@ -15,6 +17,7 @@ type HTTPRequester interface {
 type BasicClient struct {
 	Options *ClientOptions
 	client  HTTPRequester
+	uaName  string
 }
 
 type ClientOptions struct {
@@ -25,10 +28,15 @@ type ClientOptions struct {
 }
 
 func NewBasicClient(options *ClientOptions, client HTTPRequester) *BasicClient {
-	return &BasicClient{
+	bc := &BasicClient{
 		Options: options,
 		client:  client,
 	}
+
+	parsedUA := useragent.Parse(options.UserAgent)
+	bc.uaName = parsedUA.Name
+
+	return bc
 }
 
 // Makes a request with the method specified in the method parameter to the specified URL.
@@ -97,7 +105,7 @@ func (c *BasicClient) do(req *http.Request) (*ClientResponse, error) {
 	return cr, nil
 }
 
-// GetUA returns the user-agent set for this client.
-func (c *BasicClient) GetUA() string {
-	return c.Options.UserAgent
+// GetUAName returns the user-agent name for this client.
+func (c *BasicClient) GetUAName() string {
+	return c.uaName
 }

internal/crawler/basic_client_test.go

@@ -145,3 +145,29 @@ func TestHTTPError(t *testing.T) {
 		t.Fatal("expected an error, got none")
 	}
 }
+
+// Test UA name
+func TestUAName(t *testing.T) {
+	table := []struct {
+		UA   string
+		Name string
+	}{
+		{"Mozilla/5.0 (compatible; SEOnautBot/1.0; +https://seonaut.org/bot)", "SEOnautBot"},
+		{"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36", "Googlebot"},
+		{"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/W.X.Y.Z Safari/537.36", "Bingbot"},
+	}
+
+	for _, e := range table {
+		options := &crawler.ClientOptions{
+			UserAgent: e.UA,
+		}
+
+		mockClient := &mockClient{ForceError: true}
+		client := crawler.NewBasicClient(options, mockClient)
+
+		if client.GetUAName() != e.Name {
+			t.Errorf("Expected UA name: %s got: %s", e.Name, client.GetUAName())
+		}
+
+	}
+}

internal/crawler/crawler.go

@@ -36,7 +36,7 @@ var ErrDomainNotAllowed = errors.New("domain not allowed")
 type Client interface {
 	Get(urlStr string) (*ClientResponse, error)
 	Head(urlStr string) (*ClientResponse, error)
-	GetUA() string
+	GetUAName() string
 }
 
 type ResponseCallback func(r *ResponseMessage)

internal/crawler/robots_checker.go

@@ -34,7 +34,7 @@ func (r *RobotsChecker) IsBlocked(u *url.URL) bool {
 		path += "?" + u.Query().Encode()
 	}
 
-	return !robot.TestAgent(path, r.client.GetUA())
+	return !robot.TestAgent(path, r.client.GetUAName())
 }
 
 // Returns true if the robots.txt file exists and is valid

internal/crawler/robots_checker_test.go

@@ -20,7 +20,7 @@ func (t *MockClient) Get(u string) (*crawler.ClientResponse, error) {
 	r := &http.Response{}
 	if strings.HasPrefix(u, "https://example.com/") {
 		body := `
-		User-Agent: *
+		User-Agent: SEOnautBot
 		Disallow: /disallowed
 		Sitemap: /sitemap.xml
 		`
@@ -34,8 +34,8 @@ func (t *MockClient) Get(u string) (*crawler.ClientResponse, error) {
 	return &crawler.ClientResponse{Response: r}, nil
 }
 
-func (t *MockClient) GetUA() string {
-	return "TEST UA"
+func (t *MockClient) GetUAName() string {
+	return "SEOnautBot"
 }
 
 // TestIsBlocked tests URLs allowed and disallowed in the robots.txt file.