Merge pull request #242 from setlog/feature-log-fingerprint

Log fingerprint

Author: dragetd

CHANGES.md

@@ -2,6 +2,7 @@
 
 ## TBD (TBD)
 ### Changes
+* Every TLS Certificate fingerprint will be logged once with the host name it has first been seen on.
 * DWARF symbols are now stripped from the trivrost binary to reduce file size. This can save a few bytes on some platforms.
 * The binary is now compressed with UPX when using `make compress`. Reduces the final filesize to less than 50%.
 * Shorter log-output for proxy detection. Reduces average size of the log output by 5–15%.

pkg/fetching/download.go

@@ -80,6 +80,9 @@ type Download struct {
 
 	response       *http.Response
 	responseReader io.Reader
+
+	// Communicate some TLS information to the downloader which is managing this download.
+	downloader *Downloader
 }
 
 func NewDownload(ctx context.Context, resourceUrl string) *Download {
@@ -139,7 +142,7 @@ func (dl *Download) Close() error {
 func (dl *Download) readDownload(p []byte) (bytesReadCount int, err error) {
 	if dl.response == nil {
 		dl.request, dl.cancelRequest = dl.createRequest()
-		dl.response = dl.sendRequest(dl.request)
+		dl.sendRequest(dl.request)
 		if dl.response == nil {
 			return 0, nil
 		}
@@ -159,18 +162,22 @@ func (dl *Download) createRequest() (*http.Request, context.CancelFunc) {
 	return newRangeRequestWithCancel(dl.ctx, dl.url, dl.firstByteIndex, dl.lastByteIndex)
 }
 
-func (dl *Download) sendRequest(req *http.Request) *http.Response {
+func (dl *Download) sendRequest(req *http.Request) {
 	resp, err := DoForClientFunc(dl.client, req)
 	if err != nil {
 		dl.cleanUp()
 		dl.handler.HandleHttpGetError(dl.url, err)
+		dl.response = nil
 		dl.inscribeCooldown()
 	} else {
+		dl.response = resp
+		if dl.downloader != nil {
+			dl.downloader.downloadInitiatedSuccessfully(dl)
+		}
 		counter := &writeCounter{counted: uint64(dl.firstByteIndex), url: dl.url, workerId: dl.workerId, handler: dl.handler}
 		timeoutingBodyReader := &TimeoutingReader{Reader: resp.Body, Timeout: defaultTimeout * 30}
 		dl.responseReader = io.TeeReader(timeoutingBodyReader, counter)
 	}
-	return resp
 }
 
 func (dl *Download) processResponse() {

pkg/fetching/downloader.go

@@ -3,6 +3,7 @@ package fetching
 import (
 	"context"
 	"crypto/rsa"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
@@ -31,13 +32,30 @@ const MaxConcurrentDownloads = 5
 // Downloader has helper functions for common use cases of Download, such as writing a resource to a file while downloading it,
 // downloading multiple resources in parallel and verifying the hashsum or signature of downloading resources.
 type Downloader struct {
-	handler DownloadProgressHandler
-	client  *http.Client
-	ctx     context.Context
+	handler          DownloadProgressHandler
+	client           *http.Client
+	ctx              context.Context
+	seenFingerprints map[string]bool
 }
 
 func NewDownloader(ctx context.Context, handler DownloadProgressHandler) *Downloader {
-	return &Downloader{handler: handler, client: MakeClient(), ctx: ctx}
+	return &Downloader{handler: handler, client: MakeClient(), ctx: ctx, seenFingerprints: make(map[string]bool)}
+}
+
+func (downloader *Downloader) downloadInitiatedSuccessfully(dl *Download) {
+	if dl.response.TLS == nil {
+		return
+	}
+	if len(dl.response.TLS.PeerCertificates) == 0 {
+		return
+	}
+	cert := dl.response.TLS.PeerCertificates[0]
+	sha1Sum := sha1.Sum(cert.Raw)
+	sha1SumHex := hex.EncodeToString(sha1Sum[:])
+	if _, ok := downloader.seenFingerprints[sha1SumHex]; !ok {
+		downloader.seenFingerprints[sha1SumHex] = true
+		log.Printf("Seeing new fingerprint %s (sha1) for host %v", sha1SumHex, dl.request.Host)
+	}
 }
 
 func (downloader *Downloader) DownloadSignedResource(fromURL string, keys []*rsa.PublicKey) ([]byte, error) {
@@ -74,33 +92,6 @@ func (downloader *Downloader) DownloadSignedResources(urls []string, keys []*rsa
 	return validatedResources, nil
 }
 
-func (downloader *Downloader) DownloadBytes(fromURL string) (data []byte) {
-	success := false
-	var err error
-	for !success {
-		dl := downloader.newDownload(fromURL)
-		data, err = ioutil.ReadAll(dl)
-		if err != nil {
-			log.Printf("Download of \"%s\" failed: %v", fromURL, err)
-		}
-		if downloader.ctx.Err() != nil {
-			panic(downloader.ctx.Err())
-		}
-		success = err == nil
-	}
-	return
-}
-
-func (downloader *Downloader) newDownload(resourceUrl string) *Download {
-	return &Download{
-		url:      resourceUrl,
-		client:   downloader.client,
-		ctx:      downloader.ctx,
-		handler:  downloader.handler,
-		workerId: 0,
-	}
-}
-
 func (downloader *Downloader) MustDownloadToTempDirectory(baseUrl string, fileMap config.FileInfoMap, localDirPath string) (tempDirectoryPath string) {
 	reachedEndOfFunction := false // https://stackoverflow.com/a/34851179/10513183
 	tempDirectoryPath = system.MustMakeTempDirectory(localDirPath)
@@ -200,6 +191,7 @@ func (downloader *Downloader) runDownloadWorkers(ctx context.Context, cancelFunc
 			break
 		case workerId := <-availableWorkerIds:
 			dl := NewDownloadForConcurrentUse(ctx, url, downloader.client, downloader.handler, workerId)
+			dl.downloader = downloader
 			go downloadWorker(dl, availableWorkerIds, allWorkersDoneCond, workerErrChan, processDownload)
 		}
 	}