Merge branch 'master' into mta-sts-upgrade

Author: sydneyli

.travis.yml

@@ -10,7 +10,7 @@ env:
   - TEST_DB_NAME=starttls_test
 
 install:
-  - go get -u github.com/golang/lint/golint
+  - go get -u golang.org/x/lint/golint
   - go get github.com/mattn/goveralls
   - go get -t ./...
 

api.go

@@ -180,9 +180,9 @@ func getDomainParams(r *http.Request) (models.Domain, error) {
 	}
 	mtasts := r.FormValue("mta-sts")
 	domain := models.Domain{
-		Name:       name,
-		MTASTSMode: mtasts,
-		State:      models.StateUnvalidated,
+		Name:   name,
+		MTASTS: mtasts == "on",
+		State:  models.StateUnconfirmed,
 	}
 	email, err := getParam("email", r)
 	if err == nil {
@@ -233,7 +233,7 @@ func (api API) Queue(r *http.Request) APIResponse {
 		if err != nil {
 			return badRequest(err.Error())
 		}
-		ok, msg, scan := domain.IsQueueable(api.Database, api.List)
+		ok, msg, scan := domain.IsQueueable(api.Database, api.Database, api.List)
 		if !ok {
 			return badRequest(msg)
 		}
@@ -257,13 +257,13 @@ func (api API) Queue(r *http.Request) APIResponse {
 		if err != nil {
 			return badRequest(err.Error())
 		}
-		status, err := api.Database.GetDomain(domainName)
+		domainObj, err := models.GetDomain(api.Database, domainName)
 		if err != nil {
 			return APIResponse{StatusCode: http.StatusNotFound, Message: err.Error()}
 		}
 		return APIResponse{
 			StatusCode: http.StatusOK,
-			Response:   status,
+			Response:   domainObj,
 		}
 	}
 	return APIResponse{StatusCode: http.StatusMethodNotAllowed,

checker/Dockerfile

@@ -0,0 +1,11 @@
+FROM golang:1.11-alpine
+
+WORKDIR /go/src/github.com/EFForg/starttls-backend/checker
+
+RUN apk add git
+
+ADD . .
+
+RUN go get github.com/EFForg/starttls-backend/checker/cmd/starttls-check
+
+CMD ["/go/bin/starttls-check"]

checker/checker.go

@@ -20,8 +20,9 @@ type Checker struct {
 	// domain. It is used to mock DNS lookups during testing.
 	lookupMXOverride func(string) ([]*net.MX, error)
 
-	// checkHostnameOverride is used to mock checks for a single hostname.
-	checkHostnameOverride func(string, string) HostnameResult
+	// CheckHostname defines the function that should be used to check each hostname.
+	// If nil, FullCheckHostname (all hostname checks) will be used.
+	CheckHostname func(string, string, time.Duration) HostnameResult
 
 	// checkMTASTSOverride is used to mock MTA-STS checks.
 	checkMTASTSOverride func(string, map[string]HostnameResult) *MTASTSResult

checker/cmd/starttls-check/cmd.go

@@ -83,6 +83,9 @@ func main() {
 
 	domainReader := csv.NewReader(instream)
 	if *aggregate {
+		c = checker.Checker{
+			CheckHostname: checker.NoopCheckHostname,
+		}
 		resultHandler = &checker.DomainTotals{
 			Time:   time.Now(),
 			Source: label,

checker/cmd/starttls-check/cmd_test.go

@@ -17,9 +17,9 @@ import (
 func TestUpdateStats(t *testing.T) {
 	out = new(bytes.Buffer)
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintln(w, `1,foo,example1.com
-2,bar,example2.com
-3,baz,example3.com`)
+		fmt.Fprintln(w, `1,foo,localhost
+2,bar,localhost
+3,baz,localhost`)
 	}))
 	defer ts.Close()
 

checker/domain.go

@@ -120,7 +120,7 @@ func (c *Checker) CheckDomain(domain string, expectedHostnames []string) DomainR
 	}
 	checkedHostnames := make([]string, 0)
 	for _, hostname := range hostnames {
-		hostnameResult := c.CheckHostnameWithCache(domain, hostname)
+		hostnameResult := c.checkHostname(domain, hostname)
 		result.HostnameResults[hostname] = hostnameResult
 		if hostnameResult.couldConnect() {
 			checkedHostnames = append(checkedHostnames, hostname)

checker/domain_test.go

@@ -59,7 +59,7 @@ func mockLookupMX(domain string) ([]*net.MX, error) {
 	return result, nil
 }
 
-func mockCheckHostname(domain string, hostname string) HostnameResult {
+func mockCheckHostname(domain string, hostname string, _ time.Duration) HostnameResult {
 	if result, ok := hostnameResults[hostname]; ok {
 		return HostnameResult{
 			Result:    &result,
@@ -110,11 +110,11 @@ func performTests(t *testing.T, tests []domainTestCase) {
 
 func performTestsWithCacheTimeout(t *testing.T, tests []domainTestCase, cacheExpiry time.Duration) {
 	c := Checker{
-		Timeout:               time.Second,
-		Cache:                 MakeSimpleCache(cacheExpiry),
-		lookupMXOverride:      mockLookupMX,
-		checkHostnameOverride: mockCheckHostname,
-		checkMTASTSOverride:   mockCheckMTASTS,
+		Timeout:             time.Second,
+		Cache:               MakeSimpleCache(cacheExpiry),
+		lookupMXOverride:    mockLookupMX,
+		CheckHostname:       mockCheckHostname,
+		checkMTASTSOverride: mockCheckMTASTS,
 	}
 	for _, test := range tests {
 		if test.expectedHostnames == nil {

checker/hostname.go

@@ -210,29 +210,41 @@ func checkTLSVersion(client *smtp.Client, hostname string, timeout time.Duration
 	return result.Success()
 }
 
-// CheckHostnameWithCache returns the result of CheckHostname, using or
-// updating the Checker's cache.
-func (c *Checker) CheckHostnameWithCache(domain string, hostname string) HostnameResult {
+// checkHostname returns the result of c.CheckHostname or FullCheckHostname,
+// using or updating the Checker's cache.
+func (c *Checker) checkHostname(domain string, hostname string) HostnameResult {
+	check := c.CheckHostname
+	if check == nil {
+		// If CheckHostname hasn't been set, default to the full set of checks.
+		check = FullCheckHostname
+	}
+
 	if c.Cache == nil {
-		return c.CheckHostname(domain, hostname)
+		return check(domain, hostname, c.timeout())
 	}
 	hostnameResult, err := c.Cache.GetHostnameScan(hostname)
 	if err != nil {
-		hostnameResult = c.CheckHostname(domain, hostname)
+		hostnameResult = check(domain, hostname, c.timeout())
 		c.Cache.PutHostnameScan(hostname, hostnameResult)
 	}
 	return hostnameResult
 }
 
-// CheckHostname performs a series of checks against a hostname for an email domain.
-// `domain` is the mail domain that this server serves email for.
-// `hostname` is the hostname for this server.
-func (c *Checker) CheckHostname(domain string, hostname string) HostnameResult {
-	if c.checkHostnameOverride != nil {
-		// Allow the Checker to mock this function.
-		return c.checkHostnameOverride(domain, hostname)
+// NoopCheckHostname returns a fake error result containing `domain` and `hostname`.
+func NoopCheckHostname(domain string, hostname string, _ time.Duration) HostnameResult {
+	r := HostnameResult{
+		Domain:   domain,
+		Hostname: hostname,
+		Result:   MakeResult("hostnames"),
 	}
+	r.addCheck(MakeResult(Connectivity).Error("Skipping hostname checks"))
+	return r
+}
 
+// FullCheckHostname performs a series of checks against a hostname for an email domain.
+// `domain` is the mail domain that this server serves email for.
+// `hostname` is the hostname for this server.
+func FullCheckHostname(domain string, hostname string, timeout time.Duration) HostnameResult {
 	result := HostnameResult{
 		Domain:    domain,
 		Hostname:  hostname,
@@ -242,7 +254,7 @@ func (c *Checker) CheckHostname(domain string, hostname string) HostnameResult {
 
 	// Connect to the SMTP server and use that connection to perform as many checks as possible.
 	connectivityResult := MakeResult(Connectivity)
-	client, err := smtpDialWithTimeout(hostname, c.timeout())
+	client, err := smtpDialWithTimeout(hostname, timeout)
 	if err != nil {
 		result.addCheck(connectivityResult.Error("Could not establish connection: %v", err))
 		return result
@@ -258,7 +270,7 @@ func (c *Checker) CheckHostname(domain string, hostname string) HostnameResult {
 	// result.addCheck(checkTLSCipher(hostname))
 
 	// Creates a new connection to check for SSLv2/3 support because we can't call starttls twice.
-	result.addCheck(checkTLSVersion(client, hostname, c.timeout()))
+	result.addCheck(checkTLSVersion(client, hostname, timeout))
 
 	return result
 }

checker/hostname_test.go

@@ -25,8 +25,6 @@ func TestMain(m *testing.M) {
 
 const testTimeout = 250 * time.Millisecond
 
-var testChecker = Checker{Timeout: testTimeout}
-
 // Code follows pattern from crypto/tls/generate_cert.go
 // to generate a cert from a PEM-encoded RSA private key.
 func createCert(keyData string, commonName string) string {
@@ -102,7 +100,7 @@ func TestPolicyMatch(t *testing.T) {
 }
 
 func TestNoConnection(t *testing.T) {
-	result := testChecker.CheckHostname("", "example.com")
+	result := FullCheckHostname("", "example.com", testTimeout)
 
 	expected := Result{
 		Status: 3,
@@ -117,7 +115,7 @@ func TestNoTLS(t *testing.T) {
 	ln := smtpListenAndServe(t, &tls.Config{})
 	defer ln.Close()
 
-	result := testChecker.CheckHostname("", ln.Addr().String())
+	result := FullCheckHostname("", ln.Addr().String(), testTimeout)
 
 	expected := Result{
 		Status: 2,
@@ -137,7 +135,7 @@ func TestSelfSigned(t *testing.T) {
 	ln := smtpListenAndServe(t, &tls.Config{Certificates: []tls.Certificate{cert}})
 	defer ln.Close()
 
-	result := testChecker.CheckHostname("", ln.Addr().String())
+	result := FullCheckHostname("", ln.Addr().String(), testTimeout)
 
 	expected := Result{
 		Status: 2,
@@ -163,7 +161,7 @@ func TestNoTLS12(t *testing.T) {
 	})
 	defer ln.Close()
 
-	result := testChecker.CheckHostname("", ln.Addr().String())
+	result := FullCheckHostname("", ln.Addr().String(), testTimeout)
 
 	expected := Result{
 		Status: 2,
@@ -196,7 +194,7 @@ func TestSuccessWithFakeCA(t *testing.T) {
 	// conserving the port number.
 	addrParts := strings.Split(ln.Addr().String(), ":")
 	port := addrParts[len(addrParts)-1]
-	result := testChecker.CheckHostname("", "localhost:"+port)
+	result := FullCheckHostname("", "localhost:"+port, testTimeout)
 	expected := Result{
 		Status: 0,
 		Checks: map[string]*Result{
@@ -271,7 +269,7 @@ func TestFailureWithBadHostname(t *testing.T) {
 	// conserving the port number.
 	addrParts := strings.Split(ln.Addr().String(), ":")
 	port := addrParts[len(addrParts)-1]
-	result := testChecker.CheckHostname("", "localhost:"+port)
+	result := FullCheckHostname("", "localhost:"+port, testTimeout)
 	expected := Result{
 		Status: 2,
 		Checks: map[string]*Result{
@@ -311,7 +309,7 @@ func TestAdvertisedCiphers(t *testing.T) {
 
 	ln := smtpListenAndServe(t, tlsConfig)
 	defer ln.Close()
-	testChecker.CheckHostname("", ln.Addr().String())
+	FullCheckHostname("", ln.Addr().String(), testTimeout)
 
 	// Partial list of ciphers we want to support
 	expectedCipherSuites := []struct {