Add email alert to SELinux jobs

Enhance the kops test generator to optionaly add email alerts + configure
them for the SELinux jobs

Author: jsafrane

config/jobs/kubernetes/kops/build_jobs.py

@@ -85,7 +85,9 @@ def build_test(cloud='aws',
                build_cluster=None,
                cluster_name=None,
                template_path=None,
-               storage_e2e_cred=False):
+               storage_e2e_cred=False,
+               alert_email=None,
+               alert_num_failures=None):
     # pylint: disable=too-many-statements,too-many-branches,too-many-arguments
     if kops_version is None:
         kops_deploy_url = marker_updown_green(None)
@@ -259,6 +261,11 @@ def build_test(cloud='aws',
         'testgrid-days-of-results': str(days_of_results),
         'testgrid-tab-name': tab,
     }
+    if alert_email is not None:
+        annotations['testgrid-alert-email'] = alert_email
+    if alert_num_failures is not None:
+        annotations['testgrid-num-failures-to-alert'] = str(alert_num_failures)
+
     for (k, v) in spec.items():
         annotations[f"test.kops.k8s.io/{k}"] = v or ""
 
@@ -299,7 +306,9 @@ def presubmit_test(branch='master',
                    use_boskos=False,
                    build_cluster=None,
                    cluster_name=None,
-                   use_preset_for_account_creds=None):
+                   use_preset_for_account_creds=None,
+                   alert_email=None,
+                   alert_num_failures=None):
     # pylint: disable=too-many-statements,too-many-branches,too-many-arguments
     if cloud == 'aws':
         if distro == "channels":
@@ -422,6 +431,11 @@ def presubmit_test(branch='master',
         'testgrid-days-of-results': '90',
         'testgrid-tab-name': tab_name or name,
     }
+    if alert_email is not None:
+        annotations['testgrid-alert-email'] = alert_email
+    if alert_num_failures is not None:
+        annotations['testgrid-num-failures-to-alert'] = str(alert_num_failures)
+
     for (k, v) in spec.items():
         annotations[f"test.kops.k8s.io/{k}"] = v or ""
 
@@ -932,7 +946,9 @@ def generate_misc():
                    test_args="--master-os-distro=custom --node-os-distro=custom",
                    # Serial and Disruptive tests can be slow.
                    test_timeout_minutes=120,
-                   runs_per_day=3),
+                   runs_per_day=3,
+                   alert_email="[email protected]",
+                   alert_num_failures=10),
 
         # [sig-storage, @jsafrane] A one-off scenario testing all SELinux related feature gates enabled
         # and opt-in selinux-warning-controller.
@@ -970,7 +986,9 @@ def generate_misc():
                    test_args="--master-os-distro=custom --node-os-distro=custom",
                    # Serial and Disruptive tests can be slow.
                    test_timeout_minutes=120,
-                   runs_per_day=3),
+                   runs_per_day=3,
+                   alert_email="[email protected]",
+                   alert_num_failures=10),
 
         # test kube-up to kops jobs migration
         build_test(name_override="ci-kubernetes-e2e-cos-gce-canary",

config/jobs/kubernetes/kops/kops-periodics-misc2.yaml

@@ -2145,8 +2145,10 @@ periodics:
     test.kops.k8s.io/kops_channel: alpha
     test.kops.k8s.io/kops_version: latest
     test.kops.k8s.io/networking: cilium
+    testgrid-alert-email: [email protected]
     testgrid-dashboards: kops-distro-rhel9, kops-k8s-ci, kops-latest, sig-cluster-lifecycle-kops
     testgrid-days-of-results: '90'
+    testgrid-num-failures-to-alert: '10'
     testgrid-tab-name: kops-aws-selinux
 
 # {"cloud": "aws", "distro": "rhel9", "extra_flags": "--set=cluster.spec.containerd.selinuxEnabled=true --set=cluster.spec.kubeControllerManager.controllers=* --set=cluster.spec.kubeControllerManager.controllers=selinux-warning-controller --discovery-store=s3://k8s-kops-prow/discovery", "feature_flags": "SELinuxMount", "k8s_version": "ci", "kops_channel": "alpha", "kops_version": "latest", "networking": "cilium"}
@@ -2216,8 +2218,10 @@ periodics:
     test.kops.k8s.io/kops_channel: alpha
     test.kops.k8s.io/kops_version: latest
     test.kops.k8s.io/networking: cilium
+    testgrid-alert-email: [email protected]
     testgrid-dashboards: kops-distro-rhel9, kops-k8s-ci, kops-latest, sig-cluster-lifecycle-kops
     testgrid-days-of-results: '90'
+    testgrid-num-failures-to-alert: '10'
     testgrid-tab-name: kops-aws-selinux-alpha
 
 # {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-370-67 --set=spec.nodeProblemDetector.enabled=true --gce-service-account=default", "k8s_version": "ci", "kops_channel": "alpha", "kops_version": "latest", "networking": "kubenet"}