Merge pull request openstack-k8s-operators#233 from kstrenkova/add-expected-failures-list

Introduce new parameter Expected Failures List

Author: openshift-merge-bot[bot]

.zuul.yaml

@@ -57,6 +57,8 @@
         tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON
         tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_rescope_token
         tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_update_user_password
+      cifmw_test_operator_tempest_expected_failures_list: |
+        foobar
       cifmw_tempest_tempestconf_config:
         overrides: |
           compute-feature-enabled.dhcp_domain ''

api/bases/test.openstack.org_tempests.yaml

@@ -168,6 +168,12 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: The expectedFailuresList parameter contains tests
+                      that should not count as failures. When a test from this list
+                      fails, the test pod ends with Completed state rather than with
+                      Error state.
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +609,12 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: The expectedFailuresList parameter contains
+                            tests that should not count as failures. When a test from
+                            this list fails, the test pod ends with Completed state
+                            rather than with Error state.
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

api/v1beta1/tempest_types.go

@@ -151,6 +151,13 @@ type TempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList string `json:"excludeList"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The expectedFailuresList parameter contains tests that should not count
+	// as failures. When a test from this list fails, the test pod ends with
+	// Completed state rather than with Error state.
+	ExpectedFailuresList string `json:"expectedFailuresList"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// +kubebuilder:default:=0

api/v1beta1/tempest_types_workflow.go

@@ -30,6 +30,13 @@ type WorkflowTempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList *string `json:"excludeList,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The expectedFailuresList parameter contains tests that should not count
+	// as failures. When a test from this list fails, the test pod ends with
+	// Completed state rather than with Error state.
+	ExpectedFailuresList *string `json:"expectedFailuresList,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// Concurrency value that is passed to tempest via --concurrency

api/v1beta1/zz_generated.deepcopy.go

@@ -168,6 +168,12 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: The expectedFailuresList parameter contains tests
+                      that should not count as failures. When a test from this list
+                      fails, the test pod ends with Completed state rather than with
+                      Error state.
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +609,12 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: The expectedFailuresList parameter contains
+                            tests that should not count as failures. When a test from
+                            this list fails, the test pod ends with Completed state
+                            rather than with Error state.
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

config/crd/bases/test.openstack.org_tempests.yaml

@@ -46,6 +46,8 @@ spec:
     concurrency: 8
     # excludeList: | # <-- Use | to preserve \n
     #   tempest.api.identity.v3.*
+    # expectedFailuresList: | # <-- Use | to preserve \n
+    #   tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token
     # workerFile: | # <-- Use | to preserve \n
     #   - worker:
     #     - tempest.api.*

config/samples/test_v1beta1_tempest.yaml

@@ -473,6 +473,13 @@ func (r *TempestReconciler) setTempestConfigVars(envVars map[string]string,
 		envVars["TEMPEST_EXCLUDE_LIST"] = testOperatorDir + excludeListFile
 	}
 
+	value = mergeWithWorkflow(tRun.ExpectedFailuresList, wtRun.ExpectedFailuresList)
+	if len(value) != 0 {
+		expectedFailuresListFile := "expected_failures.txt"
+		customData[expectedFailuresListFile] = value
+		envVars["TEMPEST_EXPECTED_FAILURES_LIST"] = testOperatorDir + expectedFailuresListFile
+	}
+
 	// Bool
 	tempestBoolEnvVars := map[string]bool{
 		"TEMPEST_SERIAL":     mergeWithWorkflow(tRun.Serial, wtRun.Serial),