Merge pull request #1130 from SgtCoDFish/trivy-changes

Run trivy tests more often and improve reporting

Author: cert-manager-prow[bot]

config/jobs/cert-manager/cert-manager/master/cert-manager-master.yaml

@@ -1028,9 +1028,10 @@ periodics:
   annotations:
     description: Runs a Trivy scan against the controller container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-master
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1061,16 +1062,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: master
-  cron: 48 11-23/24 * * *
+  cron: 48 02-23/12 * * *
 - name: ci-cert-manager-master-trivy-test-acmesolver
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the acmesolver container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-master
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1101,16 +1103,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: master
-  cron: 52 18-23/24 * * *
+  cron: 52 04-23/12 * * *
 - name: ci-cert-manager-master-trivy-test-startupapicheck
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the startupapicheck container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-master
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1141,16 +1144,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: master
-  cron: 56 01-23/24 * * *
+  cron: 56 06-23/12 * * *
 - name: ci-cert-manager-master-trivy-test-cainjector
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the cainjector container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-master
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1181,16 +1185,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: master
-  cron: 00 08-23/24 * * *
+  cron: 00 08-23/12 * * *
 - name: ci-cert-manager-master-trivy-test-webhook
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the webhook container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-master
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1221,4 +1226,4 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: master
-  cron: 04 15-23/24 * * *
+  cron: 04 10-23/12 * * *

config/jobs/cert-manager/cert-manager/release-1.18/cert-manager-release-1.18.yaml

@@ -1123,9 +1123,10 @@ periodics:
   annotations:
     description: Runs a Trivy scan against the controller container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.18
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1156,16 +1157,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.18
-  cron: 57 19-23/24 * * *
+  cron: 57 03-23/12 * * *
 - name: ci-cert-manager-release-1.18-trivy-test-acmesolver
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the acmesolver container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.18
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1196,16 +1198,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.18
-  cron: 01 02-23/24 * * *
+  cron: 01 05-23/12 * * *
 - name: ci-cert-manager-release-1.18-trivy-test-startupapicheck
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the startupapicheck container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.18
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1236,16 +1239,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.18
-  cron: 05 09-23/24 * * *
+  cron: 05 07-23/12 * * *
 - name: ci-cert-manager-release-1.18-trivy-test-cainjector
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the cainjector container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.18
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1276,16 +1280,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.18
-  cron: 09 16-23/24 * * *
+  cron: 09 09-23/12 * * *
 - name: ci-cert-manager-release-1.18-trivy-test-webhook
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the webhook container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.18
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1316,4 +1321,4 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.18
-  cron: 13 23-23/24 * * *
+  cron: 13 11-23/12 * * *

config/jobs/cert-manager/cert-manager/release-1.19/cert-manager-release-1.19.yaml

@@ -992,9 +992,10 @@ periodics:
   annotations:
     description: Runs a Trivy scan against the controller container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.19
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1025,16 +1026,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.19
-  cron: 50 13-23/24 * * *
+  cron: 50 04-23/12 * * *
 - name: ci-cert-manager-release-1.19-trivy-test-acmesolver
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the acmesolver container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.19
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1065,16 +1067,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.19
-  cron: 54 20-23/24 * * *
+  cron: 54 06-23/12 * * *
 - name: ci-cert-manager-release-1.19-trivy-test-startupapicheck
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the startupapicheck container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.19
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1105,16 +1108,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.19
-  cron: 58 03-23/24 * * *
+  cron: 58 08-23/12 * * *
 - name: ci-cert-manager-release-1.19-trivy-test-cainjector
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the cainjector container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.19
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1145,16 +1149,17 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.19
-  cron: 02 10-23/24 * * *
+  cron: 02 10-23/12 * * *
 - name: ci-cert-manager-release-1.19-trivy-test-webhook
   max_concurrency: 2
   decorate: true
   annotations:
     description: Runs a Trivy scan against the webhook container
     testgrid-alert-email: [email protected]
-    testgrid-alert-stale-results-hours: "36"
+    testgrid-alert-stale-results-hours: "18"
     testgrid-create-job-group: "true"
     testgrid-dashboards: cert-manager-periodics-release-1.19
+    testgrid-num-columns-recent: "1"
     testgrid-num-failures-to-alert: "1"
   labels:
     preset-dind-enabled: "true"
@@ -1185,4 +1190,4 @@ periodics:
   - org: cert-manager
     repo: cert-manager
     base_ref: release-1.19
-  cron: 06 17-23/24 * * *
+  cron: 06 00-23/12 * * *

config/prowgen/pkg/configurers.go

@@ -147,6 +147,15 @@ func addTestGridCustomFailuresToAlert(failuresToAlert int) JobConfigurer {
 	}
 }
 
+// addTestGridNumColumnsRecent changes the number of test results to considered when testgrid
+// decides whether a test is "flaky"
+// See https://github.com/kubernetes/test-infra/blob/737791c6e2ee79bdc8efce2195eb6d20ebb6eb04/testgrid/config.md#prow-job-configuration
+func addTestGridNumColumnsRecent(numColumnsRecent int) JobConfigurer {
+	return func(job *Job) {
+		job.Annotations["testgrid-num-columns-recent"] = fmt.Sprintf("%d", numColumnsRecent)
+	}
+}
+
 // addTestGridStaleResultsAlert sets, in hours, the length of time before a job should be
 // considered stale. This guards against a job not running for whatever reason.
 func addTestGridStaleResultsAlert(hoursUntilStale int) JobConfigurer {

config/prowgen/pkg/generators.go

@@ -19,6 +19,7 @@ package pkg
 
 import (
 	"fmt"
+	"math"
 	"strings"
 )
 
@@ -328,9 +329,12 @@ func UpgradeTest(ctx *ProwContext, k8sVersion string) *Job {
 // so e.g. if there's a vuln in the "controller" container we might never scan "ctl" container.
 // Instead, we generate a test for each container so it's obvious which ones have failures and it's easier to get results
 // for each container
-func TrivyTest(ctx *ProwContext, containerName string) *Job {
+// periodicity is the number of hours between runs of this job; this is used to calculate when the job should be considered stale
+func TrivyTest(ctx *ProwContext, containerName string, periodicity int) *Job {
 	containerName = strings.ToLower(containerName)
 
+	stale := math.Round(float64(periodicity) * 1.5)
+
 	job := jobTemplate(
 		fmt.Sprintf("trivy-test-%s", containerName),
 		fmt.Sprintf("Runs a Trivy scan against the %s container", containerName),
@@ -342,9 +346,13 @@ func TrivyTest(ctx *ProwContext, containerName string) *Job {
 		// Need to ensure that trivy tests send a failure email as soon as they fail since
 		// they tend to be run relatively infrequently and a failure is important to address
 		addTestGridCustomFailuresToAlert(1),
+		// Trivy tests are quite binary - either the scan passes or fails.
+		// Having a fixed test report as "flaky" isn't helpful, so set "num columns recent" to 1
+		// so that the test should report as either passing or failing but not flaky.
+		addTestGridNumColumnsRecent(1),
 		// Ask TestGrid to alert us if the job hasn't run in the last 36 hours. Sets
 		// an upper limit on how regularly the job can be scheduled.
-		addTestGridStaleResultsAlert(36),
+		addTestGridStaleResultsAlert(int(stale)),
 	)
 
 	makeJobs, cpuRequest := calculateMakeConcurrency("1000m")

config/prowgen/prowspecs/specs.go

@@ -167,7 +167,8 @@ func (m *BranchSpec) GenerateJobFile() *pkg.JobFile {
 	}
 
 	for _, container := range m.containerNames {
-		m.prowContext.Periodics(pkg.TrivyTest(m.prowContext, container), 24)
+		periodicity := 12
+		m.prowContext.Periodics(pkg.TrivyTest(m.prowContext, container, periodicity), periodicity)
 	}
 
 	return m.prowContext.JobFile()