[az][load] bug fixes on kvrefid splitcsv & terminal state (#8297)

* [az][load] bug fixes on kvrefid splitcsv & terminal state

* [az][load] bug fixes on kvrefid splitcsv & terminal state

* refactor for pylint R0912, version bump

* update HISTORY text

* add terminal state for aio

Author: mbhardwaj-msft

src/load/HISTORY.rst

@@ -2,6 +2,13 @@
 
 Release History
 ===============
+1.3.1
+++++++
+* Bug fix for `splitAllCSVs` not being honoured from config file due to CLI argument being set as false by default leading to configuration not being selected from the config file.
+* Bug fix for `keyVaultReferenceIdentity` not being honoured from config file as the key being looked up while YAML parsing was incorrect.
+* Change 'VALIDATION_FAILED' to 'VALIDATION_FAILURE' as a terminal status for File Validation.
+* Add 'NOT_VALIDATED' as a terminal status for File Validation in Async IO.
+
 1.3.0
 ++++++
 * Add support for autostop criteria. Autostop error rate and time window in seconds can be set using `--autostop-error-rate` and `--autostop-time-window` arguments in 'az load test create' and 'az load test update' commands. Autostop can be disabled by using `--autostop disable` in 'az load test create' and 'az load test update' commands. Autostop criteria set in YAML config file will now also be honoured.

src/load/azext_load/data_plane/load_test/custom.py

@@ -60,8 +60,6 @@ def create_test(
         raise InvalidArgumentValueError(msg)
     body = {}
     yaml, yaml_test_body = None, None
-    if split_csv is None:
-        split_csv = False
     autostop_criteria = create_autostop_criteria_from_args(
         autostop=autostop, error_rate=autostop_error_rate, time_window=autostop_error_rate_time_window)
     if load_test_config_file is None:

src/load/azext_load/data_plane/utils/utils.py

@@ -4,12 +4,11 @@
 # --------------------------------------------------------------------------------------------
 
 import os
-import uuid
 from enum import EnumMeta
 
 import requests
 import yaml
-from azext_load.data_plane.utils import validators
+from azext_load.data_plane.utils import validators, utils_yaml_config
 from azext_load.vendored_sdks.loadtesting_mgmt import LoadTestMgmtClient
 from azure.cli.core.azclierror import (
     FileOperationError,
@@ -296,8 +295,8 @@ def convert_yaml_to_test(data):
     if "description" in data:
         new_body["description"] = data["description"]
     new_body["keyvaultReferenceIdentityType"] = IdentityType.SystemAssigned
-    if "keyvaultReferenceIdentityId" in data:
-        new_body["keyvaultReferenceIdentityId"] = data["keyvaultReferenceIdentityId"]
+    if "keyVaultReferenceIdentity" in data:
+        new_body["keyvaultReferenceIdentityId"] = data["keyVaultReferenceIdentity"]
         new_body["keyvaultReferenceIdentityType"] = IdentityType.UserAssigned
 
     if "subnetId" in data:
@@ -322,62 +321,11 @@ def convert_yaml_to_test(data):
             "Quick start test is not supported currently in CLI. Please use portal to run quick start test"
         )
     if data.get("splitAllCSVs") is not None:
-        new_body["loadTestConfiguration"]["splitAllCSVs"] = data.get("splitAllCSVs")
-
+        new_body["loadTestConfiguration"]["splitAllCSVs"] = utils_yaml_config.yaml_parse_splitcsv(data=data)
     if data.get("failureCriteria"):
-        new_body["passFailCriteria"] = {}
-        new_body["passFailCriteria"]["passFailMetrics"] = {}
-        for items in data["failureCriteria"]:
-            metric_id = get_random_uuid()
-            # check if item is string or dict. if string then no name is provided
-            name = None
-            components = items
-            if isinstance(items, dict):
-                name = list(items.keys())[0]
-                components = list(items.values())[0]
-            # validate failure criteria
-            try:
-                validate_failure_criteria(components)
-            except InvalidArgumentValueError as e:
-                logger.error("Invalid failure criteria: %s", str(e))
-            new_body["passFailCriteria"]["passFailMetrics"][metric_id] = {}
-            new_body["passFailCriteria"]["passFailMetrics"][metric_id]["aggregate"] = (
-                components.split("(")[0].strip()
-            )
-            new_body["passFailCriteria"]["passFailMetrics"][metric_id][
-                "clientMetric"
-            ] = (components.split("(")[1].split(")")[0].strip())
-            new_body["passFailCriteria"]["passFailMetrics"][metric_id]["condition"] = (
-                components.split(")")[1].strip()[0]
-            )
-            new_body["passFailCriteria"]["passFailMetrics"][metric_id]["value"] = (
-                components.split(
-                    new_body["passFailCriteria"]["passFailMetrics"][metric_id]["condition"]
-                )[1].strip()
-            )
-            if name is not None:
-                new_body["passFailCriteria"]["passFailMetrics"][metric_id][
-                    "requestName"
-                ] = name
+        new_body["passFailCriteria"] = utils_yaml_config.yaml_parse_failure_criteria(data=data)
     if data.get("autoStop") is not None:
-        if (isinstance(data["autoStop"], str)):
-            # pylint: disable-next=protected-access
-            validators._validate_autostop_disable_configfile(data["autoStop"])
-            new_body["autoStopCriteria"] = {
-                "autoStopDisabled": True,
-            }
-        else:
-            error_rate = data["autoStop"].get("errorPercentage")
-            time_window = data["autoStop"].get("timeWindow")
-            # pylint: disable-next=protected-access
-            validators._validate_autostop_criteria_configfile(error_rate, time_window)
-            new_body["autoStopCriteria"] = {
-                "autoStopDisabled": False,
-            }
-            if error_rate is not None:
-                new_body["autoStopCriteria"]["errorRate"] = error_rate
-            if time_window is not None:
-                new_body["autoStopCriteria"]["errorRateTimeWindowInSeconds"] = time_window
+        new_body["autoStopCriteria"] = utils_yaml_config.yaml_parse_autostop_criteria(data=data)
     logger.debug("Converted yaml to test body: %s", new_body)
     return new_body
 
@@ -421,7 +369,7 @@ def create_or_update_test_with_config(
         new_body["keyvaultReferenceIdentityType"] = IdentityType.UserAssigned
     elif yaml_test_body.get("keyvaultReferenceIdentityId") is not None:
         new_body["keyvaultReferenceIdentityId"] = yaml_test_body.get(
-            "keyVaultReferenceIdentity"
+            "keyvaultReferenceIdentityId"
         )
         new_body["keyvaultReferenceIdentityType"] = IdentityType.UserAssigned
     else:
@@ -802,20 +750,3 @@ def upload_files_helper(
         client=client,
         test_id=test_id, yaml_data=yaml_data, test_plan=test_plan,
         load_test_config_file=load_test_config_file, existing_test_files=files, wait=wait)
-
-
-def validate_failure_criteria(failure_criteria):
-    parts = failure_criteria.split("(")
-    if len(parts) != 2:
-        raise ValueError(f"Invalid failure criteria: {failure_criteria}")
-    _, condition_value = parts
-    if (
-        ")" not in condition_value
-        or len(condition_value.split(")")) != 2
-        or condition_value.endswith(")")
-    ):
-        raise ValueError(f"Invalid failure criteria: {failure_criteria}")
-
-
-def get_random_uuid():
-    return str(uuid.uuid4())

src/load/azext_load/data_plane/utils/utils_yaml_config.py

@@ -0,0 +1,99 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import uuid
+
+from azext_load.data_plane.utils import validators
+from azure.cli.core.azclierror import (
+    InvalidArgumentValueError,
+)
+
+from knack.log import get_logger
+
+logger = get_logger(__name__)
+
+
+def yaml_parse_autostop_criteria(data):
+    if (isinstance(data["autoStop"], str)):
+        # pylint: disable-next=protected-access
+        validators._validate_autostop_disable_configfile(data["autoStop"])
+        return {
+            "autoStopDisabled": True,
+        }
+    error_rate = data["autoStop"].get("errorPercentage")
+    time_window = data["autoStop"].get("timeWindow")
+    # pylint: disable-next=protected-access
+    validators._validate_autostop_criteria_configfile(error_rate, time_window)
+    autostop_criteria = {
+        "autoStopDisabled": False,
+    }
+    if error_rate is not None:
+        autostop_criteria["errorRate"] = error_rate
+    if time_window is not None:
+        autostop_criteria["errorRateTimeWindowInSeconds"] = time_window
+    return autostop_criteria
+
+
+def yaml_parse_splitcsv(data):
+    if not isinstance(data.get("splitAllCSVs"), bool):
+        raise InvalidArgumentValueError(
+            "Invalid value for splitAllCSVs. Allowed values are boolean true or false"
+        )
+    return data.get("splitAllCSVs")
+
+
+def validate_failure_criteria(failure_criteria):
+    parts = failure_criteria.split("(")
+    if len(parts) != 2:
+        raise ValueError(f"Invalid failure criteria: {failure_criteria}")
+    _, condition_value = parts
+    if (
+        ")" not in condition_value
+        or len(condition_value.split(")")) != 2
+        or condition_value.endswith(")")
+    ):
+        raise ValueError(f"Invalid failure criteria: {failure_criteria}")
+
+
+def get_random_uuid():
+    return str(uuid.uuid4())
+
+
+def yaml_parse_failure_criteria(data):
+    passfail_criteria = {}
+    passfail_criteria["passFailMetrics"] = {}
+    for items in data["failureCriteria"]:
+        metric_id = get_random_uuid()
+        # check if item is string or dict. if string then no name is provided
+        name = None
+        components = items
+        if isinstance(items, dict):
+            name = list(items.keys())[0]
+            components = list(items.values())[0]
+        # validate failure criteria
+        try:
+            validate_failure_criteria(components)
+        except InvalidArgumentValueError as e:
+            logger.error("Invalid failure criteria: %s", str(e))
+        passfail_criteria["passFailMetrics"][metric_id] = {}
+        passfail_criteria["passFailMetrics"][metric_id]["aggregate"] = (
+            components.split("(")[0].strip()
+        )
+        passfail_criteria["passFailMetrics"][metric_id][
+            "clientMetric"
+        ] = (components.split("(")[1].split(")")[0].strip())
+        passfail_criteria["passFailMetrics"][metric_id]["condition"] = (
+            components.split(")")[1].strip()[0]
+        )
+        passfail_criteria["passFailMetrics"][metric_id]["value"] = (
+            components.split(
+                passfail_criteria["passFailMetrics"][metric_id]["condition"]
+            )[1].strip()
+        )
+        if name is not None:
+            passfail_criteria["passFailMetrics"][metric_id][
+                "requestName"
+            ] = name
+    return passfail_criteria

src/load/azext_load/tests/latest/constants.py

@@ -19,6 +19,10 @@ class LoadConstants:
     INVALID_ZIP_ARTIFACT_LOAD_TEST_CONFIG_FILE = os.path.join(
         TEST_RESOURCES_DIR, r"invalid-config-with-zip-artifacts.yaml"
     )
+    LOAD_TEST_CONFIG_FILE_KVREFID = os.path.join(TEST_RESOURCES_DIR, r"config-kvrefid.yaml")
+    LOAD_TEST_CONFIG_FILE_INVALID_KVREFID = os.path.join(TEST_RESOURCES_DIR, r"config-invalid-kvrefid.yaml")
+    LOAD_TEST_CONFIG_FILE_SPLITCSV_FALSE = os.path.join(TEST_RESOURCES_DIR, r"config-splitcsv-false.yaml")
+    LOAD_TEST_CONFIG_FILE_INVALID_SPLITCSV = os.path.join(TEST_RESOURCES_DIR, r"config-invalid-splitcsv.yaml")
     TEST_PLAN = os.path.join(TEST_RESOURCES_DIR, r"sample-JMX-file.jmx")
     ADDITIONAL_FILE = os.path.join(TEST_RESOURCES_DIR, r"additional-data.csv")
     FILE_NAME = "sample-JMX-file.jmx"
@@ -98,6 +102,8 @@ class LoadTestConstants(LoadConstants):
     APP_COMPONENT_TEST_ID = "app-component-test-case"
     SERVER_METRIC_TEST_ID = "server-metric-test-case"
     FILE_TEST_ID = "file-test-case"
+    LOAD_TEST_KVREF_ID = "loadtest-kvrefid-case"
+    LOAD_TEST_SPLITCSV_ID = "loadtest-splitcsv-case"
 
     INVALID_UPDATE_TEST_ID = "invalid-update-test-case"
     INVALID_PF_TEST_ID = "invalid-pf-test-case"