Merge pull request #71 from sc0rp10/main

#21 implement the result cache

Author: smartinov

README.md

@@ -82,16 +82,17 @@ Configuration specification in JSON and plain is supported both in command line
 
 Configuration of targets can be done via docker and via prometheus
 
-| Flag             | Variable             | Description                                   | Default                                          | Required |
-|------------------|----------------------|-----------------------------------------------|--------------------------------------------------|----------|
-| -api-key         | PAGESPEED_API_KEY    | sets the google API key used for pagespeed    |                                                  | False    |
-| -targets         | PAGESPEED_TARGETS    | comma separated list of targets to measure    |                                                  | False    |
-| -categories      | PAGESPEED_CATEGORIES | comma separated list of categories to check   | accessibility,best-practices,performance,pwa,seo | False    |
-| -t               | NONE                 | multi-value target array (check docker comp)  |                                                  | False    |
-| -listener        | PAGESPEED_LISTENER   | sets the listener address for the exporters   | :9271                                            | False    |
-| -parallel        | PAGESPEED_PARALLEL   | sets the execution of targets to be parallel  | false                                            | False    |
-| -pushGatewayUrl  | PUSHGATEWAY_URL      | sets the pushgateway url to send the metrics  |                                                  | False    |
-| -pushGatewayJob  | PUSHGATEWAY_JOB      | sets the pushgateway job name                 | pagespeed_exporter                               | False    |
+| Flag             | Variable             | Description                                                       | Default                                          | Required |
+|------------------|----------------------|-----------------------------------------------|----------------------------------------------------------------------|----------|
+| -api-key         | PAGESPEED_API_KEY    | sets the google API key used for pagespeed                        |                                                  | False    |
+| -targets         | PAGESPEED_TARGETS    | comma separated list of targets to measure                        |                                                  | False    |
+| -categories      | PAGESPEED_CATEGORIES | comma separated list of categories to check                       | accessibility,best-practices,performance,pwa,seo | False    |
+| -t               | NONE                 | multi-value target array (check docker comp)                      |                                                  | False    |
+| -listener        | PAGESPEED_LISTENER   | sets the listener address for the exporters                       | :9271                                            | False    |
+| -parallel        | PAGESPEED_PARALLEL   | sets the execution of targets to be parallel                      | false                                            | False    |
+| -pushGatewayUrl  | PUSHGATEWAY_URL      | sets the pushgateway url to send the metrics                      |                                                  | False    |
+| -pushGatewayJob  | PUSHGATEWAY_JOB      | sets the pushgateway job name                                     | pagespeed_exporter                               | False    |
+| -cache-ttl       | CACHE_TTL            | cache TTL for API results (e.g. 60s, 5m); disables cache if unset |                                                  | False    |
 
 Note: google api key is required only if scraping more than 2 targets/second
 

collector/cache.go

@@ -0,0 +1,63 @@
+package collector
+
+import (
+	"sync"
+	"time"
+	"crypto/sha256"
+	"encoding/json"
+)
+
+type cacheEntry struct {
+	Result    *ScrapeResult
+	ExpiresAt time.Time
+}
+
+type scrapeCache struct {
+	entries map[string]cacheEntry
+	mutex   sync.Mutex
+	ttl     time.Duration
+}
+
+func newScrapeCache(ttl time.Duration) *scrapeCache {
+	if ttl <= 0 {
+		return nil
+	}
+	return &scrapeCache{
+		entries: make(map[string]cacheEntry),
+		ttl:     ttl,
+	}
+}
+
+func (c *scrapeCache) get(key string) (*ScrapeResult, bool) {
+	if c == nil {
+		return nil, false
+	}
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	entry, ok := c.entries[key]
+	if !ok || time.Now().After(entry.ExpiresAt) {
+		if ok {
+			delete(c.entries, key)
+		}
+		return nil, false
+	}
+	return entry.Result, true
+}
+
+func (c *scrapeCache) set(key string, result *ScrapeResult) {
+	if c == nil {
+		return
+	}
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.entries[key] = cacheEntry{
+		Result:    result,
+		ExpiresAt: time.Now().Add(c.ttl),
+	}
+}
+
+func cacheKeyFromRequest(req ScrapeRequest) string {
+	b, _ := json.Marshal(req)
+	h := sha256.Sum256(b)
+	return string(h[:])
+}

collector/collector.go

@@ -70,7 +70,7 @@ func newCollector(config Config) (coll prometheus.Collector, err error) {
 		options = append(options, option.WithCredentialsFile(config.CredentialsFile))
 	}
 
-	svc, err := newPagespeedScrapeService(config.ScrapeTimeout, options...)
+	svc, err := newPagespeedScrapeService(config.ScrapeTimeout, config.CacheTTL, options...)
 	if err != nil {
 		return nil, err
 	}

collector/model.go

@@ -78,6 +78,7 @@ type Config struct {
 	CredentialsFile string
 	Parallel        bool
 	ScrapeTimeout   time.Duration
+	CacheTTL        time.Duration // cache duration, 0 disables cache
 }
 
 func CalculateScrapeRequests(targets, categories []string) []ScrapeRequest {

collector/scrape.go

@@ -23,7 +23,7 @@ type scrapeService interface {
 
 // newPagespeedScrapeService creates a new HTTP client service for pagespeed.
 // If the client timeout is set to 0 there will be no timeout
-func newPagespeedScrapeService(clientTimeout time.Duration, options ...option.ClientOption) (scrapeService, error) {
+func newPagespeedScrapeService(clientTimeout time.Duration, cacheTTL time.Duration, options ...option.ClientOption) (scrapeService, error) {
 	transport, err := googlehttp.NewTransport(context.Background(), http.DefaultTransport, options...)
 	if err != nil {
 		return nil, err
@@ -40,12 +40,14 @@ func newPagespeedScrapeService(clientTimeout time.Duration, options ...option.Cl
 	return &pagespeedScrapeService{
 		scrapeClient: client,
 		options:      options,
+		cache:        newScrapeCache(cacheTTL),
 	}, nil
 }
 
 type pagespeedScrapeService struct {
 	scrapeClient *http.Client
 	options      []option.ClientOption
+	cache        *scrapeCache
 }
 
 func (pss *pagespeedScrapeService) Scrape(parallel bool, requests []ScrapeRequest) (scrapes []*ScrapeResult, err error) {
@@ -99,6 +101,12 @@ func (pss *pagespeedScrapeService) Scrape(parallel bool, requests []ScrapeReques
 }
 
 func (pss pagespeedScrapeService) scrape(request ScrapeRequest) (scrape *ScrapeResult, err error) {
+	cacheKey := cacheKeyFromRequest(request)
+	if pss.cache != nil {
+		if cached, ok := pss.cache.get(cacheKey); ok {
+			return cached, nil
+		}
+	}
 	opts := []option.ClientOption{
 		option.WithHTTPClient(pss.scrapeClient),
 	}
@@ -134,8 +142,12 @@ func (pss pagespeedScrapeService) scrape(request ScrapeRequest) (scrape *ScrapeR
 		return nil, errResult
 	}
 
-	return &ScrapeResult{
+	scrapeResult := &ScrapeResult{
 		Request: request,
 		Result:  result,
-	}, nil
+	}
+	if pss.cache != nil {
+		pss.cache.set(cacheKey, scrapeResult)
+	}
+	return scrapeResult, nil
 }

collector/scrape_test.go

@@ -32,7 +32,7 @@ func Test_PagespeedScrapeService(t *testing.T) {
 		t.Skip("skipping testing unless API key or credentials file is set")
 	}
 
-	service, err := newPagespeedScrapeService(30*time.Second, options...)
+	service, err := newPagespeedScrapeService(30*time.Second, 0, options...) // cache disabled for test
 	if err != nil {
 		t.Fatalf("newPagespeedScrapeService should not throw an error: %v", err)
 	}

pagespeed_exporter.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/foomo/pagespeed_exporter/collector"
 	"github.com/foomo/pagespeed_exporter/handler"
@@ -24,6 +25,7 @@ var (
 	parallel        bool
 	pushGatewayUrl  string
 	pushGatewayJob  string
+	cacheTTL        string // as duration string, e.g. "60s"
 )
 
 type arrayFlags []string
@@ -47,11 +49,22 @@ func main() {
 	if len(targets) > 0 {
 		requests := collector.CalculateScrapeRequests(targets, categories)
 
+		var parsedCacheTTL time.Duration
+		if cacheTTL != "" {
+			var err error
+			parsedCacheTTL, err = time.ParseDuration(cacheTTL)
+			if err != nil {
+				log.WithError(err).Warn("invalid CACHE_TTL, disabling cache")
+				parsedCacheTTL = 0
+			}
+		}
+
 		psc, errCollector := collectorFactory.Create(collector.Config{
 			ScrapeRequests:  requests,
 			GoogleAPIKey:    googleApiKey,
 			CredentialsFile: credentialsFile,
 			Parallel:        parallel,
+			CacheTTL:        parsedCacheTTL,
 		})
 		if errCollector != nil {
 			log.WithError(errCollector).Fatal("could not instantiate collector")
@@ -73,6 +86,7 @@ func main() {
 }
 
 func parseFlags() {
+	flag.StringVar(&cacheTTL, "cache-ttl", getenv("CACHE_TTL", ""), "cache TTL for API results, e.g. 60s. If empty, disables cache")
 	flag.StringVar(&googleApiKey, "api-key", getenv("PAGESPEED_API_KEY", ""), "sets the google API key used for pagespeed")
 	flag.StringVar(&credentialsFile, "credentials-file", getenv("PAGESPEED_CREDENTIALS_FILE", ""), "sets the location of the credentials file used for pagespeed")
 	flag.StringVar(&listenerAddress, "listener", getenv("PAGESPEED_LISTENER", ":9271"), "sets the listener address for the exporters")