Skip to content

Commit 8442695

Browse files
penmanglewoodpenmanglewood
authored
Parameterize pagespeed categories (#45)
This creates an environment variable and CLI flag to set which pagespeed categories to fetch. By default, it fetches all five, preserving this package's default functionality. If categories are specified in a target as JSON, it will supercede the categories specified via the envvar or flag.
1 parent 78700a4 commit 8442695

File tree

9 files changed

+218
-70
lines changed

9 files changed

+218
-70
lines changed

‎README.md‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ The provided dashboard (Pagespeed) will be loaded with data after the first scra
2020

2121
The dashboard can be found at [grafana](https://grafana.com/dashboards/9510)
2222

23+
Note: The example dashboard assumes you're fetching all pagespeed categories.
24+
2325
## Understanding Metrics
2426

2527
* https://github.com/GoogleChrome/lighthouse/blob/master/docs/understanding-results.md
@@ -62,12 +64,16 @@ Or via JSON which adds additional parameters
6264
// URL can't be invalid
6365
// Strategy can only be mobile/desktop
6466
// If strategy is not specified, both desktop & mobile will be used
67+
// Categories can be any of accessibility/best-practices/performance/pwa/seo
68+
// If categories are not specified, all categories will be used
6569
// Parameters are passed down to google pagespeed api
6670
6771
{"url":"https://github.com/foomo/pagespeed_exporter","campaign":"test","locale":"en","source":"source"}
6872
6973
{"url":"https://mysite.com/test?test=true","strategy":"mobile"}
7074
75+
{"url":"https://mysite.com/test?test=true","categories": ["best-practices"]}
76+
7177
```
7278

7379
Configuration specification in JSON and plain is supported both in command line & prometheus configuration
@@ -76,15 +82,16 @@ Configuration specification in JSON and plain is supported both in command line
7682

7783
Configuration of targets can be done via docker and via prometheus
7884

79-
| Flag | Variable | Description | Default | Required |
80-
|------------------|--------------------|-----------------------------------------------|--------------------|----------|
81-
| -api-key | PAGESPEED_API_KEY | sets the google API key used for pagespeed | | False |
82-
| -targets | PAGESPEED_TARGETS | comma separated list of targets to measure | | False |
83-
| -t | NONE | multi-value target array (check docker comp) | | False |
84-
| -listener | PAGESPEED_LISTENER | sets the listener address for the exporters | :9271 | False |
85-
| -parallel | PAGESPEED_PARALLEL | sets the execution of targets to be parallel | false | False |
86-
| -pushGatewayUrl | PUSHGATEWAY_URL | sets the pushgateway url to send the metrics | | False |
87-
| -pushGatewayJob | PUSHGATEWAY_JOB | sets the pushgateway job name | pagespeed_exporter | False |
85+
| Flag | Variable | Description | Default | Required |
86+
|------------------|----------------------|-----------------------------------------------|--------------------------------------------------|----------|
87+
| -api-key | PAGESPEED_API_KEY | sets the google API key used for pagespeed | | False |
88+
| -targets | PAGESPEED_TARGETS | comma separated list of targets to measure | | False |
89+
| -categories | PAGESPEED_CATEGORIES | comma separated list of categories to check | accessibility,best-practices,performance,pwa,seo | False |
90+
| -t | NONE | multi-value target array (check docker comp) | | False |
91+
| -listener | PAGESPEED_LISTENER | sets the listener address for the exporters | :9271 | False |
92+
| -parallel | PAGESPEED_PARALLEL | sets the execution of targets to be parallel | false | False |
93+
| -pushGatewayUrl | PUSHGATEWAY_URL | sets the pushgateway url to send the metrics | | False |
94+
| -pushGatewayJob | PUSHGATEWAY_JOB | sets the pushgateway job name | pagespeed_exporter | False |
8895

8996
Note: google api key is required only if scraping more than 2 targets/second
9097

@@ -137,6 +144,7 @@ or
137144
$ docker run -p "9271:9271" --rm \
138145
--env PAGESPEED_API_KEY={KEY} \
139146
--env PAGESPEED_TARGETS=https://google.com,https://prometheus.io \
147+
--env PAGESPEED_CATEGORIES=accessibility,pwa \
140148
foomo/pagespeed_exporter
141149
```
142150

‎collector/collector.go‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ var (
1818
_ Factory = factory{}
1919
)
2020

21+
var (
22+
timeValueRe = regexp.MustCompile(`(\d*[.]?\d+(ms|s))|0`)
23+
timeUnitRe = regexp.MustCompile(`(ms|s)`)
24+
)
25+
2126
type Factory interface {
2227
Create(config Config) (prometheus.Collector, error)
2328
}
@@ -124,7 +129,7 @@ func collect(scrape *ScrapeResult, ch chan<- prometheus.Metric) error {
124129
}
125130

126131
if r.LighthouseResult != nil {
127-
collectLighthouseResults("lighthouse", r.LighthouseResult, constLabels, ch)
132+
collectLighthouseResults("lighthouse", scrape.Request.Categories, r.LighthouseResult, constLabels, ch)
128133
}
129134
return nil
130135
}
@@ -162,46 +167,44 @@ func collectLoadingExperience(prefix string, lexp *pagespeedonline.PagespeedApiL
162167

163168
}
164169

165-
func collectLighthouseResults(prefix string, lhr *pagespeedonline.LighthouseResultV5, constLabels prometheus.Labels, ch chan<- prometheus.Metric) {
170+
func collectLighthouseResults(prefix string, cats []string, lhr *pagespeedonline.LighthouseResultV5, constLabels prometheus.Labels, ch chan<- prometheus.Metric) {
166171

167172
ch <- prometheus.MustNewConstMetric(
168173
prometheus.NewDesc(fqname(prefix, "total_duration_seconds"), "The total time spent in seconds loading the page and evaluating audits.", nil, constLabels),
169174
prometheus.GaugeValue,
170175
lhr.Timing.Total/1000) //ms -> seconds
171176

172177
categories := map[string]*pagespeedonline.LighthouseCategoryV5{
173-
"performance": lhr.Categories.Performance,
174-
"accessibility": lhr.Categories.Accessibility,
175-
"pwa": lhr.Categories.Pwa,
176-
"best-practices": lhr.Categories.BestPractices,
177-
"seo": lhr.Categories.Seo,
178+
CategoryPerformance: lhr.Categories.Performance,
179+
CategoryAccessibility: lhr.Categories.Accessibility,
180+
CategoryPWA: lhr.Categories.Pwa,
181+
CategoryBestPractices: lhr.Categories.BestPractices,
182+
CategorySEO: lhr.Categories.Seo,
178183
}
179184

180-
for k, v := range categories {
181-
score, err := strconv.ParseFloat(fmt.Sprint(v.Score), 64)
185+
for _, c := range cats {
186+
score, err := strconv.ParseFloat(fmt.Sprint(categories[c].Score), 64)
182187
if err != nil {
188+
logrus.WithError(err).Warn("could not parse category score")
183189
continue
184190
}
185191

186192
ch <- prometheus.MustNewConstMetric(
187193
prometheus.NewDesc(fqname(prefix, "category_score"), "Lighthouse score for the specified category", []string{"category"}, constLabels),
188194
prometheus.GaugeValue,
189195
score,
190-
k)
196+
c)
191197
}
192198

193199
for k, v := range lhr.Audits {
194-
re := regexp.MustCompile(`(\d*[.]?\d+(ms|s))|0`)
195-
timeRe := regexp.MustCompile(`(ms|s)`)
196-
197200
if timeAuditMetrics[k] {
198201
displayValue := strings.Replace(v.DisplayValue, "\u00a0", "", -1)
199202
displayValue = strings.Replace(displayValue, ",", "", -1)
200-
if !timeRe.MatchString(displayValue) {
203+
if !timeUnitRe.MatchString(displayValue) {
201204
displayValue = displayValue + "s"
202205
}
203206

204-
if duration, errDuration := time.ParseDuration(re.FindString(displayValue)); errDuration == nil {
207+
if duration, errDuration := time.ParseDuration(timeValueRe.FindString(displayValue)); errDuration == nil {
205208
ch <- prometheus.MustNewConstMetric(
206209
prometheus.NewDesc(fqname(prefix, k, "duration_seconds"), v.Description, nil, constLabels),
207210
prometheus.GaugeValue,

‎collector/model.go‎

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ const (
1212
StrategyMobile = Strategy("mobile")
1313
StrategyDesktop = Strategy("desktop")
1414

15+
CategoryAccessibility = "accessibility"
16+
CategoryBestPractices = "best-practices"
17+
CategorySEO = "seo"
18+
CategoryPWA = "pwa"
19+
CategoryPerformance = "performance"
20+
1521
Namespace = "pagespeed"
1622
)
1723

@@ -22,17 +28,26 @@ var availableStrategies = map[Strategy]bool{
2228
StrategyDesktop: true,
2329
}
2430

31+
var availableCategories = map[string]bool{
32+
CategoryAccessibility: true,
33+
CategoryBestPractices: true,
34+
CategorySEO: true,
35+
CategoryPWA: true,
36+
CategoryPerformance: true,
37+
}
38+
2539
type ScrapeResult struct {
2640
Request ScrapeRequest
2741
Result *pagespeedonline.PagespeedApiPagespeedResponseV5
2842
}
2943

3044
type ScrapeRequest struct {
31-
Url string `json:"url"`
32-
Strategy Strategy `json:"strategy"`
33-
Campaign string `json:"campaign"`
34-
Source string `json:"source"`
35-
Locale string `json:"locale"`
45+
Url string `json:"url"`
46+
Strategy Strategy `json:"strategy"`
47+
Campaign string `json:"campaign"`
48+
Source string `json:"source"`
49+
Locale string `json:"locale"`
50+
Categories []string `json:"categories"`
3651
}
3752

3853
func (sr ScrapeRequest) IsValid() bool {
@@ -43,6 +58,13 @@ func (sr ScrapeRequest) IsValid() bool {
4358
if !availableStrategies[sr.Strategy] {
4459
return false
4560
}
61+
62+
for _, c := range sr.Categories {
63+
if !availableCategories[c] {
64+
return false
65+
}
66+
}
67+
4668
if _, err := url.ParseRequestURI(sr.Url); err != nil {
4769
return false
4870
}
@@ -58,7 +80,7 @@ type Config struct {
5880
ScrapeTimeout time.Duration
5981
}
6082

61-
func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
83+
func CalculateScrapeRequests(targets, categories []string) []ScrapeRequest {
6284
if len(targets) == 0 {
6385
return nil
6486
}
@@ -67,6 +89,7 @@ func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
6789
for _, t := range targets {
6890
var request ScrapeRequest
6991
if err := json.Unmarshal([]byte(t), &request); err == nil {
92+
populateCategories(&request, categories)
7093
if request.Strategy != "" {
7194
requests = append(requests, request)
7295
} else {
@@ -77,10 +100,11 @@ func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
77100
requests = append(requests, desktop, mobile)
78101
}
79102
} else {
80-
requests = append(requests,
81-
ScrapeRequest{Url: t, Strategy: StrategyDesktop},
82-
ScrapeRequest{Url: t, Strategy: StrategyMobile},
83-
)
103+
desktop := ScrapeRequest{Url: t, Strategy: StrategyDesktop}
104+
mobile := ScrapeRequest{Url: t, Strategy: StrategyMobile}
105+
populateCategories(&desktop, categories)
106+
populateCategories(&mobile, categories)
107+
requests = append(requests, desktop, mobile)
84108
}
85109
}
86110

@@ -94,3 +118,22 @@ func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
94118

95119
return filtered
96120
}
121+
122+
// populateCategories sets categories in the scrape request if not already set
123+
func populateCategories(r *ScrapeRequest, cats []string) {
124+
if r.Categories != nil && len(r.Categories) != 0 {
125+
return
126+
}
127+
128+
if cats == nil {
129+
cats = make([]string, 0, len(availableCategories))
130+
}
131+
132+
if len(cats) == 0 {
133+
for c := range availableCategories {
134+
cats = append(cats, c)
135+
}
136+
}
137+
138+
r.Categories = cats
139+
}

0 commit comments

Comments
 (0)