Introduce new parameter Expected Failures List

The Expected Failures List parameter will help with specifying set
of tests whose failures we want to ignore. By adding test to this
list, its failure will not have impact on the jobs pass/fail. It
can help with not having to re-run long running jobs and save
time and resources in specific cases, for example when dealing
with flaky tests.

Depends-On: openstack-k8s-operators/ci-framework#2513

Author: kstrenkova

.zuul.yaml

@@ -57,6 +57,8 @@
         tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON
         tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_rescope_token
         tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_update_user_password
+      cifmw_test_operator_tempest_expected_failures_list: |
+        foobar
       cifmw_tempest_tempestconf_config:
         overrides: |
           compute-feature-enabled.dhcp_domain ''

api/bases/test.openstack.org_tempests.yaml

@@ -168,6 +168,12 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: The expectedFailuresList parameter contains tests
+                      that should not count as failures. When a test from this list
+                      fails, the test pod ends with Completed state rather than with
+                      Error state.
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +609,12 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: The expectedFailuresList parameter contains
+                            tests that should not count as failures. When a test from
+                            this list fails, the test pod ends with Completed state
+                            rather than with Error state.
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

api/v1beta1/tempest_types.go

@@ -151,6 +151,13 @@ type TempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList string `json:"excludeList"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The expectedFailuresList parameter contains tests that should not count
+	// as failures. When a test from this list fails, the test pod ends with
+	// Completed state rather than with Error state.
+	ExpectedFailuresList string `json:"expectedFailuresList"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// +kubebuilder:default:=0

api/v1beta1/tempest_types_workflow.go

@@ -30,6 +30,13 @@ type WorkflowTempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList *string `json:"excludeList,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The expectedFailuresList parameter contains tests that should not count
+	// as failures. When a test from this list fails, the test pod ends with
+	// Completed state rather than with Error state.
+	ExpectedFailuresList *string `json:"expectedFailuresList,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// Concurrency value that is passed to tempest via --concurrency

api/v1beta1/zz_generated.deepcopy.go

@@ -168,6 +168,12 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: The expectedFailuresList parameter contains tests
+                      that should not count as failures. When a test from this list
+                      fails, the test pod ends with Completed state rather than with
+                      Error state.
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +609,12 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: The expectedFailuresList parameter contains
+                            tests that should not count as failures. When a test from
+                            this list fails, the test pod ends with Completed state
+                            rather than with Error state.
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

config/crd/bases/test.openstack.org_tempests.yaml

@@ -46,6 +46,8 @@ spec:
     concurrency: 8
     # excludeList: | # <-- Use | to preserve \n
     #   tempest.api.identity.v3.*
+    # expectedFailuresList: | # <-- Use | to preserve \n
+    #   tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token
     # workerFile: | # <-- Use | to preserve \n
     #   - worker:
     #     - tempest.api.*

config/samples/test_v1beta1_tempest.yaml

@@ -444,6 +444,13 @@ func (r *TempestReconciler) setTempestConfigVars(envVars map[string]string,
 		envVars["TEMPEST_EXCLUDE_LIST"] = testOperatorDir + excludeListFile
 	}
 
+	value = mergeWithWorkflow(tRun.ExpectedFailuresList, wtRun.ExpectedFailuresList)
+	if len(value) != 0 {
+		expectedFailuresListFile := "expected_failures.txt"
+		customData[expectedFailuresListFile] = value
+		envVars["TEMPEST_EXPECTED_FAILURES_LIST"] = testOperatorDir + expectedFailuresListFile
+	}
+
 	// Bool
 	tempestBoolEnvVars := map[string]bool{
 		"TEMPEST_SERIAL":     mergeWithWorkflow(tRun.Serial, wtRun.Serial),