[load] support multi-region load | advanced url tests | dashboard reports | debug mode | copy artifacts sas url | high scale test config | convert to jmx | baseline trends | sas download (Azure#8350)

* [load] support multi-region load test configuration

* refactor

* update recordings

* azdev style load

* refactor yaml keys to constants file

* refactor test load test

* update recordings

* azdev style load

* update help text

* update help text

* action on comments

* refactor test regional-load autostop

* check for valid region names client-side

* [load] support advanced url tests

* action on comments

* action on comments

* action on comments

* update recordings

* action on comments

* update test and recordings

* refactor test exception scenarios

* azdev style load

* [load] dashboard reports | debug mode | copy artifacts sas url

* action on comments

* update recordings

* set copy artifacts ut live-only

* change copy-artifacts to get-artifacts

* update recordings

* [load] download-files high scale test update console message

* [load] download-files high scale test update console message

* remove unused import

* update texts

* update texts

* [load] convert test to jmx

* update comment

* update prompt text

* [load] baseline trends | sas download

* unit tests and recordings

* trends aggregate param,unit tests, recordings

* update HISTORY.rst

* remove faulty recording

* action copilot review

Co-authored-by: Copilot <[email protected]>

* undo faulty commit by copilot

* test instruction

* action on comments

* azdev mask recordings

---------

Co-authored-by: Copilot <[email protected]>

Author: mbhardwaj-msft

src/load/HISTORY.rst

@@ -2,6 +2,19 @@
 
 Release History
 ===============
+1.4.0
+++++++
+* Add support for multi-region load test configuration. Multi-region load test configuration can be set using `--regionwise-engines` argument in 'az load test create' and 'az load test update' commands. Multi-region load test configuration set in YAML config file under key `regionalLoadTestConfig` will also be honoured.
+* Bug fix for `engineInstances` being reset to 1 and not getting backfilled using test's existing configuration when engine instances are not explicitly specified either in YAML config file or CLI argument.
+* Add support for advanced URL test with multiple HTTP request using JSON file. Add `--test-type` argument to 'az load test create' and honor `testType` key in YAML config file.
+* Add CLI parameter `--report` to 'az load test-run download-files' to download the dashboard reports.
+* Enable debug level logging using `--debug-mode` argument in 'az load test-run create' command .
+* Return the SAS URL to copy artifacts to storage accounts using command 'az load test-run get-artifacts-url'.
+* Add config for high-scale load tests and extend 'az load test-run download-files' to support download of logs and results from artifacts container for such tests.
+* Add command 'az load test convert-to-jmx' to convert URL type tests to JMX tests.
+* Add commands 'az load test set-baseline' to set the baseline for a test and 'az load test compare-to-baseline' to compare recent test runs to the baseline test run.
+
+
 1.3.1
 ++++++
 * Bug fix for `splitAllCSVs` not being honoured from config file due to CLI argument being set as false by default leading to configuration not being selected from the config file.

src/load/azext_load/data_plane/load_test/commands.py

@@ -4,7 +4,9 @@
 # --------------------------------------------------------------------------------------------
 
 from azext_load.data_plane.utils import validators
+from azext_load.data_plane.utils.constants import LoadCommandsConstants
 from azure.cli.core.commands import CliCommandType
+from .transformers import trends_output_transformer
 
 admin_custom_sdk = CliCommandType(
     operations_tmpl="azext_load.data_plane.load_test.custom#{}"
@@ -25,6 +27,13 @@ def load_test_commands(self, _):
             "download_test_files",
             validator=validators.validate_download,
         )
+        g.custom_command(
+            "convert-to-jmx",
+            "convert_to_jmx",
+            confirmation=LoadCommandsConstants.CONVERT_TO_JMX_CONFIRM_PROMPT
+        )
+        g.custom_command("set-baseline", "set_baseline")
+        g.custom_command("compare-to-baseline", "compare_to_baseline", table_transformer=trends_output_transformer)
 
     with self.command_group(
         "load test app-component",

src/load/azext_load/data_plane/load_test/custom.py

@@ -9,16 +9,22 @@
 
 from azext_load.data_plane.utils.utils import (
     convert_yaml_to_test,
+    create_autostop_criteria_from_args,
     create_or_update_test_with_config,
     create_or_update_test_without_config,
     download_file,
+    generate_trends_row,
     get_admin_data_plane_client,
+    get_testrun_data_plane_client,
     load_yaml,
     upload_file_to_test,
     upload_files_helper,
-    create_autostop_criteria_from_args,
 )
-from azure.cli.core.azclierror import InvalidArgumentValueError
+from azext_load.data_plane.utils.models import (
+    AllowedTestTypes,
+    AllowedTrendsResponseTimeAggregations,
+)
+from azure.cli.core.azclierror import InvalidArgumentValueError, FileOperationError
 from azure.core.exceptions import ResourceNotFoundError
 from knack.log import get_logger
 
@@ -31,6 +37,7 @@ def create_test(
     test_id,
     display_name=None,
     test_plan=None,
+    test_type=None,
     resource_group_name=None,
     load_test_config_file=None,
     test_description=None,
@@ -46,6 +53,7 @@ def create_test(
     autostop=None,
     autostop_error_rate=None,
     autostop_error_rate_time_window=None,
+    regionwise_engines=None,
 ):
     client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
     logger.info("Create test has started for test ID : %s", test_id)
@@ -68,6 +76,7 @@ def create_test(
             body,
             display_name=display_name,
             test_description=test_description,
+            test_type=test_type,
             engine_instances=engine_instances,
             env=env,
             secrets=secrets,
@@ -77,15 +86,17 @@ def create_test(
             split_csv=split_csv,
             disable_public_ip=disable_public_ip,
             autostop_criteria=autostop_criteria,
+            regionwise_engines=regionwise_engines,
         )
     else:
         yaml = load_yaml(load_test_config_file)
-        yaml_test_body = convert_yaml_to_test(yaml)
+        yaml_test_body = convert_yaml_to_test(cmd, yaml)
         body = create_or_update_test_with_config(
             test_id,
             body,
             yaml_test_body,
             display_name=display_name,
+            test_type=test_type,
             test_description=test_description,
             engine_instances=engine_instances,
             env=env,
@@ -95,16 +106,18 @@ def create_test(
             subnet_id=subnet_id,
             split_csv=split_csv,
             disable_public_ip=disable_public_ip,
-            autostop_criteria=autostop_criteria
+            autostop_criteria=autostop_criteria,
+            regionwise_engines=regionwise_engines,
         )
     logger.debug("Creating test with test ID: %s and body : %s", test_id, body)
     response = client.create_or_update_test(test_id=test_id, body=body)
     logger.debug(
         "Created test with test ID: %s and response obj is %s", test_id, response
     )
     logger.info("Uploading files to test %s", test_id)
+    evaluated_test_type = test_type or yaml_test_body.get("kind") if yaml_test_body else response.get("kind")
     upload_files_helper(
-        client, test_id, yaml, test_plan, load_test_config_file, not custom_no_wait
+        client, test_id, yaml, test_plan, load_test_config_file, not custom_no_wait, evaluated_test_type
     )
     response = client.get_test(test_id)
     logger.info("Upload files to test %s has completed", test_id)
@@ -133,6 +146,7 @@ def update_test(
     autostop=None,
     autostop_error_rate=None,
     autostop_error_rate_time_window=None,
+    regionwise_engines=None,
 ):
     client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
     logger.info("Update test has started for test ID : %s", test_id)
@@ -149,7 +163,7 @@ def update_test(
         autostop=autostop, error_rate=autostop_error_rate, time_window=autostop_error_rate_time_window)
     if load_test_config_file is not None:
         yaml = load_yaml(load_test_config_file)
-        yaml_test_body = convert_yaml_to_test(yaml)
+        yaml_test_body = convert_yaml_to_test(cmd, yaml)
         body = create_or_update_test_with_config(
             test_id,
             body,
@@ -164,7 +178,8 @@ def update_test(
             subnet_id=subnet_id,
             split_csv=split_csv,
             disable_public_ip=disable_public_ip,
-            autostop_criteria=autostop_criteria
+            autostop_criteria=autostop_criteria,
+            regionwise_engines=regionwise_engines,
         )
     else:
         body = create_or_update_test_without_config(
@@ -180,7 +195,8 @@ def update_test(
             subnet_id=subnet_id,
             split_csv=split_csv,
             disable_public_ip=disable_public_ip,
-            autostop_criteria=autostop_criteria
+            autostop_criteria=autostop_criteria,
+            regionwise_engines=regionwise_engines,
         )
     logger.info("Updating test with test ID: %s", test_id)
     response = client.create_or_update_test(test_id=test_id, body=body)
@@ -189,7 +205,7 @@ def update_test(
     )
     logger.info("Uploading files to test %s", test_id)
     upload_files_helper(
-        client, test_id, yaml, test_plan, load_test_config_file, not custom_no_wait
+        client, test_id, yaml, test_plan, load_test_config_file, not custom_no_wait, body.get("kind")
     )
     response = client.get_test(test_id)
     logger.info("Upload files to test %s has completed", test_id)
@@ -262,6 +278,117 @@ def delete_test(
     logger.info("Deleted test with test ID: %s", test_id)
 
 
+def convert_to_jmx(
+    cmd,
+    load_test_resource,
+    test_id,
+    resource_group_name=None,
+):
+    logger.info("Converting test with test ID: %s to JMX", test_id)
+    client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
+    existing_test = client.get_test(test_id)
+    if (
+        existing_test is not None
+        and existing_test.get("kind") != AllowedTestTypes.URL.value
+    ):
+        raise InvalidArgumentValueError(
+            f"Test with test ID: {test_id} is not of type URL. "
+            "Only URL type tests can be converted to JMX."
+        )
+    body = create_or_update_test_without_config(
+        test_id=test_id,
+        body=existing_test,
+        test_type=AllowedTestTypes.JMX.value,
+    )
+    response = client.create_or_update_test(test_id=test_id, body=body)
+    logger.debug("Converted test with test ID: %s to JMX", test_id)
+    return response.as_dict()
+
+
+def set_baseline(
+    cmd,
+    load_test_resource,
+    test_id,
+    test_run_id,
+    resource_group_name=None,
+):
+    logger.info("Setting baseline for test with test ID: %s", test_id)
+    test_client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
+    test_run_client = get_testrun_data_plane_client(cmd, load_test_resource, resource_group_name)
+    existing_test = test_client.get_test(test_id)
+    existing_test_run = test_run_client.get_test_run(test_run_id)
+    if existing_test_run.get("testId") != test_id:
+        raise InvalidArgumentValueError(
+            f"Test run with ID: {test_run_id} is not associated with test ID: {test_id}"
+        )
+    if existing_test_run.get("status") not in ["CANCELLED", "DONE"]:
+        raise InvalidArgumentValueError(
+            f"Test run with ID: {test_run_id} does not have a valid "
+            f"test run status {existing_test_run.get('status')}. "
+            "Valid test run status are: CANCELLED, DONE"
+        )
+    if existing_test_run.get("testRunStatistics", {}).get("Total") is None:
+        raise InvalidArgumentValueError(
+            f"Sampler statistics are not yet available for test run ID {test_run_id}. "
+            "Please try again later."
+        )
+    body = create_or_update_test_without_config(
+        test_id=test_id,
+        body=existing_test,
+        baseline_test_run_id=test_run_id,
+    )
+    response = test_client.create_or_update_test(test_id=test_id, body=body)
+    logger.debug("Set test run %s as baseline for test: %s", test_run_id, test_id)
+    return response.as_dict()
+
+
+def compare_to_baseline(
+    cmd,
+    load_test_resource,
+    test_id,
+    resource_group_name=None,
+    response_time_aggregate=AllowedTrendsResponseTimeAggregations.MEAN.value,
+):
+    logger.info("Showing test trends for test with test ID: %s", test_id)
+    test_client = get_admin_data_plane_client(cmd, load_test_resource, resource_group_name)
+    test_run_client = get_testrun_data_plane_client(cmd, load_test_resource, resource_group_name)
+    test = test_client.get_test(test_id)
+    if test.get("baselineTestRunId") is None:
+        raise InvalidArgumentValueError(
+            f"Test with ID: {test_id} does not have a baseline test run associated with it."
+        )
+    baseline_test_run_id = test.get("baselineTestRunId")
+    baseline_test_run = test_run_client.get_test_run(baseline_test_run_id)
+    all_test_runs = test_run_client.list_test_runs(
+        orderby="executedDateTime desc",
+        test_id=test_id,
+        status="CANCELLED,DONE",
+        maxpagesize=20,
+    )
+    all_test_runs = [run.as_dict() for run in all_test_runs]
+    logger.debug("Total number of test runs: %s", len(all_test_runs))
+    count = 0
+    recent_test_runs = []
+    for run in all_test_runs:
+        if (
+            run.get("testRunId") != baseline_test_run_id
+            and count < 10  # Show only 10 most recent test runs
+        ):
+            recent_test_runs.append(run)
+            count += 1
+
+    logger.debug("Number of recent test runs: %s", len(recent_test_runs))
+    rows = [
+        generate_trends_row(baseline_test_run, response_time_aggregate=response_time_aggregate)
+    ]
+    for run in recent_test_runs:
+        rows.append(
+            generate_trends_row(run, response_time_aggregate=response_time_aggregate)
+        )
+    logger.debug("Retrieved test trends: %s", rows)
+    return rows
+
+
 def add_test_app_component(
     cmd,
     load_test_resource,
@@ -403,6 +530,10 @@ def upload_test_file(
     response = upload_file_to_test(
         client, test_id, path, file_type=file_type, wait=not no_wait
     )
+    if not no_wait and response is not None and response.get("validationStatus") == "VALIDATION_FAILURE":
+        raise FileOperationError(
+            f"File upload failed due to validation failure: {response.get('validationFailureDetails')}"
+        )
     logger.debug("Upload test file response: %s", response)
     logger.info("Upload test file completed")
     return response.as_dict()

src/load/azext_load/data_plane/load_test/help.py

@@ -37,6 +37,12 @@
         az load test create --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --display-name "Sample Name" --autostop-error-rate 80.5 --autostop-time-window 120
         az load test create --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --display-name "Sample Name" --autostop disable
         az load test create --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --display-name "Sample Name" --autostop enable
+    - name: Create a test with a multi-region load configuration using region names in the format accepted by Azure Resource Manager (ARM). Ensure the specified regions are supported by Azure Load Testing. Multi-region load tests are restricted to public endpoints only.
+      text: |
+        az load test create --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --engine-instances 3 --regionwise-engines eastus=1 westus2=1 germanywestcentral=1 --test-plan sample-jmx.jmx
+    - name: Create an advanced URL test with multiple HTTP requests using a JSON file.
+      text: |
+        az load test create --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --test-plan ~/resources/sample-url-requests.json --test-type URL
 """
 
 helps[
@@ -82,6 +88,12 @@
     - name: Update the Key Vault reference identity to system assigned Managed Identity.
       text: |
         az load test update --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --keyvault-reference-id null
+    - name: Update autostop criteria.
+      text: |
+        az load test update --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --autostop-error-rate 90 --autostop-time-window 180
+    - name: Update multi-region load configuration.
+      text: |
+        az load test update --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --engine-instances 5 --regionwise-engines eastus=2 westus2=1 eastasia=2
 """
 
 helps[
@@ -95,6 +107,42 @@
         az load test delete --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id
 """
 
+helps[
+    "load test convert-to-jmx"
+] = """
+type: command
+short-summary: Convert a URL type test to JMX test.
+examples:
+    - name: Convert to JMX test.
+      text: |
+        az load test convert-to-jmx --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id
+"""
+
+helps[
+    "load test set-baseline"
+] = """
+type: command
+short-summary: Set a test run as the baseline for comparison with other runs in the test.
+examples:
+    - name: Set baseline test run.
+      text: |
+        az load test set-baseline --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --test-run-id sample-associated-test-run-id
+"""
+
+helps[
+    "load test compare-to-baseline"
+] = """
+type: command
+short-summary: Compare the sampler statistics from recent test runs with those of the baseline test run.
+examples:
+    - name: Compare recent test runs to baseline.
+      text: |
+        az load test compare-to-baseline --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id -o table
+    - name: Compare recent test runs to baseline with specific aggregation.
+      text: |
+        az load test compare-to-baseline --load-test-resource sample-alt-resource --resource-group sample-rg --test-id sample-existing-test-id --aggregation P95 -o table
+"""
+
 helps[
     "load test download-files"
 ] = """
@@ -234,4 +282,7 @@
     - name: Upload zipped artifacts to a test.
       text: |
         az load test file upload --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --path ~/Resources/sample-zip.zip --file-type ZIPPED_ARTIFACTS
+    - name: Upload URL requests JSON configuration file to a test which is of type URL.
+      text: |
+        az load test file upload --test-id sample-test-id --load-test-resource sample-alt-resource --resource-group sample-rg --path ~/Resources/sample-url-requests.json --file-type URL_TEST_CONFIG
 """