diff --git a/src/k8s-extension/HISTORY.rst b/src/k8s-extension/HISTORY.rst index 5bc7e3ca5d3..a40a7d29a41 100644 --- a/src/k8s-extension/HISTORY.rst +++ b/src/k8s-extension/HISTORY.rst @@ -2,6 +2,13 @@ Release History =============== +1.6.5 +++++++++++++++++++ +* microsoft.azuremonitor.containers.metrics: fix: simplify logic and enable correct recording rule groups for managed prom + +1.6.4 +++++++++++++++++++ +* microsoft.azuremonitor.containers.metrics: Update default region for azure monitor metrics extension in mooncake 1.6.3 ++++++++++++++++++ diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/DataProtectionKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/DataProtectionKubernetes.py index 67e9afc0d39..f88cc9b965d 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/DataProtectionKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/DataProtectionKubernetes.py @@ -9,7 +9,7 @@ from azure.cli.core.azclierror import RequiredArgumentMissingError, InvalidArgumentValueError from .DefaultExtension import DefaultExtension -from .._client_factory import cf_storage, cf_managed_clusters +from .._client_factory import cf_managed_clusters from ..vendored_sdks.models import (Extension, PatchExtension, Scope, ScopeCluster) logger = get_logger(__name__) @@ -20,6 +20,7 @@ def __init__(self): """Constants for configuration settings - Tenant Id (required) - Backup storage location (required) + - Resource Requests (optional) - Resource Limits (optional) """ self.TENANT_ID = "credentials.tenantId" @@ -27,6 +28,8 @@ def __init__(self): self.BACKUP_STORAGE_ACCOUNT_NAME = "configuration.backupStorageLocation.config.storageAccount" self.BACKUP_STORAGE_ACCOUNT_RESOURCE_GROUP = "configuration.backupStorageLocation.config.resourceGroup" self.BACKUP_STORAGE_ACCOUNT_SUBSCRIPTION = "configuration.backupStorageLocation.config.subscriptionId" + self.RESOURCE_REQUEST_CPU = "resources.requests.cpu" + self.RESOURCE_REQUEST_MEMORY = "resources.requests.memory" self.RESOURCE_LIMIT_CPU = "resources.limits.cpu" self.RESOURCE_LIMIT_MEMORY = "resources.limits.memory" self.BACKUP_STORAGE_ACCOUNT_USE_AAD = "configuration.backupStorageLocation.config.useAAD" @@ -36,6 +39,8 @@ def __init__(self): self.storage_account = "storageAccount" self.storage_account_resource_group = "storageAccountResourceGroup" self.storage_account_subscription = "storageAccountSubscriptionId" + self.cpu_request = "cpuRequest" + self.memory_request = "memoryRequest" self.cpu_limit = "cpuLimit" self.memory_limit = "memoryLimit" self.use_aad = "useAAD" @@ -46,6 +51,8 @@ def __init__(self): self.storage_account.lower(): self.BACKUP_STORAGE_ACCOUNT_NAME, self.storage_account_resource_group.lower(): self.BACKUP_STORAGE_ACCOUNT_RESOURCE_GROUP, self.storage_account_subscription.lower(): self.BACKUP_STORAGE_ACCOUNT_SUBSCRIPTION, + self.cpu_request.lower(): self.RESOURCE_REQUEST_CPU, + self.memory_request.lower(): self.RESOURCE_REQUEST_MEMORY, self.cpu_limit.lower(): self.RESOURCE_LIMIT_CPU, self.memory_limit.lower(): self.RESOURCE_LIMIT_MEMORY, self.use_aad.lower(): self.BACKUP_STORAGE_ACCOUNT_USE_AAD, @@ -83,7 +90,7 @@ def Create( plan_publisher, plan_product, ): - # Current scope of DataProtection Kubernetes Backup extension is 'cluster' #TODO: add TSGs when they are in place + # Current scope of DataProtection extension is 'cluster' #TODO: add TSGs when they are in place if scope == 'namespace': raise InvalidArgumentValueError(f"Invalid scope '{scope}'. This extension can only be installed at 'cluster' scope.") @@ -106,7 +113,7 @@ def Create( configuration_settings[self.TENANT_ID] = tenant_id if configuration_settings.get(self.BACKUP_STORAGE_ACCOUNT_USE_AAD) is None: - logger.warning("useAAD flag is not specified. Setting it to 'true'. Please provide extension MSI Storage Blob Data Contributor role to the storage account.") + logger.warning("useAAD flag is not specified. Setting it to 'true'. Please provide extension MSI Storage Blob Data Contributor role on the storage account.") configuration_settings[self.BACKUP_STORAGE_ACCOUNT_USE_AAD] = "true" if configuration_settings.get(self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI) is None: @@ -149,18 +156,20 @@ def Update( self.__validate_and_map_config(configuration_settings, validate_bsl=bsl_specified) if bsl_specified: self.__validate_backup_storage_account(cmd.cli_ctx, resource_group_name, cluster_name, configuration_settings) - # this step is for brownfield migrating to AAD + + # This step is for brownfield migrating to AAD if configuration_settings.get(self.BACKUP_STORAGE_ACCOUNT_USE_AAD) is not None and configuration_settings.get(self.BACKUP_STORAGE_ACCOUNT_USE_AAD).lower() == "true": - logger.warning("useAAD flag is set to true. Please provide extension MSI Storage Blob Data Contributor role to the storage account.") + logger.warning("useAAD flag is set to true. Please provide extension MSI Storage Blob Data Contributor role on the storage account.") if configuration_settings.get(self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI) is None: - # SA details provided in user inputs, but did not provide SA URI. - logger.warning("storageAccountURI is not populated. Setting it to the storage account URI of provided storage account") + # SA details provided in user input, but did not provide SA URI. if bsl_specified: + logger.warning("storageAccountURI is not populated. Setting it to the storage account URI of provided storage account.") configuration_settings[self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI] = self.__get_storage_account_uri(cmd.cli_ctx, configuration_settings) logger.warning(f"storageAccountURI: {configuration_settings[self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI]}") # SA details not provided in user input, SA Uri not provided in user input, and also not populated in the original extension, we populate it. elif not bsl_specified and original_extension.configuration_settings.get(self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI) is None: + logger.warning("storageAccountURI is not populated. Setting it to the storage account URI of the storage account provided during extension installation.") configuration_settings[self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI] = self.__get_storage_account_uri(cmd.cli_ctx, original_extension.configuration_settings) logger.warning(f"storageAccountURI: {configuration_settings[self.BACKUP_STORAGE_ACCOUNT_STORAGE_ACCOUNT_URI]}") @@ -177,7 +186,7 @@ def __get_tenant_id(self, cli_ctx): return cli_ctx.data['tenant_id'] def __validate_and_map_config(self, configuration_settings, validate_bsl=True): - """Validate and set configuration settings for Data Protection K8sBackup extension""" + """Validate and set configuration settings for DataProtection extension""" input_configuration_settings = dict(configuration_settings.items()) input_configuration_keys = [key.lower() for key in configuration_settings] @@ -189,14 +198,14 @@ def __validate_and_map_config(self, configuration_settings, validate_bsl=True): for key in input_configuration_settings: _key = key.lower() if _key in self.configuration_mapping: - configuration_settings[self.configuration_mapping[_key]] = configuration_settings.pop(key) + configuration_settings[self.configuration_mapping[_key]] = configuration_settings.pop(key).strip() else: configuration_settings.pop(key) logger.warning(f"Ignoring unrecognized configuration setting: {key}") def __validate_backup_storage_account(self, cli_ctx, resource_group_name, cluster_name, configuration_settings): """Validations performed on the backup storage account - - Existance of the storage account + - Existence of the storage account - Cluster and storage account are in the same location """ storage_account = self.__get_storage_account(cli_ctx, configuration_settings) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/EntraWorkloadIAM.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/EntraWorkloadIAM.py index a65ded9f18f..c0d93069118 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/EntraWorkloadIAM.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/EntraWorkloadIAM.py @@ -27,6 +27,25 @@ CONFIG_SETTINGS_HELM_TENANT_ID = 'global.workload-iam.tenantID' CONFIG_SETTINGS_HELM_JOIN_TOKEN = 'workload-iam-local-authority.localAuthorityArgs.joinToken' +def settings_dict_to_lowercase(settings): + """ + Create new dictionary where the keys of the known user settings are all lowercase (but leave the + others as they were in case they are specific settings that have to be passed to the Helm chart). + """ + all_user_settings = [CONFIG_SETTINGS_USER_TRUST_DOMAIN, CONFIG_SETTINGS_USER_TENANT_ID, + CONFIG_SETTINGS_USER_LOCAL_AUTHORITY, CONFIG_SETTINGS_USER_JOIN_TOKEN] + + if settings is None: + return dict() + + validated_settings = dict() + for key, value in settings.items(): + if key.lower() in all_user_settings: + validated_settings[key.lower()] = value + else: + validated_settings[key] = value + + return validated_settings def settings_dict_to_lowercase(settings): """ diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/deaults.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/deaults.py index 2e338217c2a..d09c0908973 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/deaults.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/deaults.py @@ -8,7 +8,7 @@ def get_default_region(cmd): cloud_name = cmd.cli_ctx.cloud.name if cloud_name.lower() == 'azurechinacloud': - raise CLIError("Azure China Cloud is not supported for the Azure Monitor Metrics addon") + return "chinanorth3" if cloud_name.lower() == 'azureusgovernment': return "usgovvirginia" return "eastus" diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/create.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/create.py index 5f022b6bb88..dd38491285b 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/create.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/create.py @@ -3,8 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import json -from ..constants import ALERTS_API, RULES_API from knack.util import CLIError +from ..constants import ALERTS_API, RULES_API +from azure.cli.command_modules.acs.azuremonitormetrics.recordingrules.common import truncate_rule_group_name # pylint: disable=line-too-long @@ -15,7 +16,18 @@ def get_recording_rules_template(cmd, azure_monitor_workspace_resource_id): url = f"{armendpoint}{azure_monitor_workspace_resource_id}/providers/microsoft.alertsManagement/alertRuleRecommendations?api-version={ALERTS_API}" r = send_raw_request(cmd.cli_ctx, "GET", url, headers=headers) data = json.loads(r.text) - return data['value'] + + filtered_templates = [ + template for template in data.get('value', []) + if template.get("properties", {}).get("alertRuleType", "").lower() == "microsoft.alertsmanagement/prometheusrulegroups" and isinstance(template.get("properties", {}).get("rulesArmTemplate", {}).get("resources"), list) and all( + isinstance(rule, dict) and "record" in rule and "expression" in rule + for resource in template["properties"]["rulesArmTemplate"]["resources"] + if resource.get("type", "").lower() == "microsoft.alertsmanagement/prometheusrulegroups" + for rule in resource.get("properties", {}).get("rules", []) + ) + ] + + return filtered_templates # pylint: disable=line-too-long @@ -39,8 +51,7 @@ def put_rules(cmd, default_rule_group_id, default_rule_group_name, mac_region, a for _ in range(3): try: headers = ['User-Agent=arc-azuremonitormetrics.put_rules.' + default_rule_group_name] - send_raw_request(cmd.cli_ctx, "PUT", url, - body=body, headers=headers) + send_raw_request(cmd.cli_ctx, "PUT", url, body=body, headers=headers) break except CLIError as e: error = e @@ -49,30 +60,30 @@ def put_rules(cmd, default_rule_group_id, default_rule_group_name, mac_region, a # pylint: disable=line-too-long -def create_rules(cmd, cluster_subscription, cluster_resource_group_name, cluster_name, azure_monitor_workspace_resource_id, mac_region): +def create_rules(cmd, cluster_subscription, cluster_resource_group_name, cluster_name, azure_monitor_workspace_resource_id, mac_region, raw_parameters): default_rules_template = get_recording_rules_template(cmd, azure_monitor_workspace_resource_id) - default_rule_group_name = "NodeRecordingRulesRuleGroup-{0}".format(cluster_name) - default_rule_group_id = "/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.AlertsManagement/prometheusRuleGroups/{2}".format( - cluster_subscription, - cluster_resource_group_name, - default_rule_group_name - ) - url = "{0}{1}?api-version={2}".format( - cmd.cli_ctx.cloud.endpoints.resource_manager, - default_rule_group_id, - RULES_API - ) - put_rules(cmd, default_rule_group_id, default_rule_group_name, mac_region, azure_monitor_workspace_resource_id, cluster_name, default_rules_template, url, True, 0) - default_rule_group_name = "KubernetesRecordingRulesRuleGroup-{0}".format(cluster_name) - default_rule_group_id = "/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.AlertsManagement/prometheusRuleGroups/{2}".format( - cluster_subscription, - cluster_resource_group_name, - default_rule_group_name - ) - url = "{0}{1}?api-version={2}".format( - cmd.cli_ctx.cloud.endpoints.resource_manager, - default_rule_group_id, - RULES_API - ) - put_rules(cmd, default_rule_group_id, default_rule_group_name, mac_region, azure_monitor_workspace_resource_id, cluster_name, default_rules_template, url, True, 1) + enable_windows_recording_rules = raw_parameters.get("enable_windows_recording_rules", False) + + for index, rule_template in enumerate(default_rules_template): + rule_name = rule_template["name"] + is_windows_rule = "win" in rule_name.lower() + + enable_rules = not (is_windows_rule and not enable_windows_recording_rules) + + rule_group_name = truncate_rule_group_name(f"{rule_template['name']}-{cluster_name}") + rule_group_id = f"/subscriptions/{cluster_subscription}/resourceGroups/{cluster_resource_group_name}/providers/Microsoft.AlertsManagement/prometheusRuleGroups/{rule_group_name}" + url = f"{cmd.cli_ctx.cloud.endpoints.resource_manager}{rule_group_id}?api-version={RULES_API}" + + put_rules( + cmd, + rule_group_id, + rule_group_name, + mac_region, + azure_monitor_workspace_resource_id, + cluster_name, + default_rules_template, + url, + enable_rules, + index + ) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/delete.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/delete.py index ac945fbaed8..bd50143f6b6 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/delete.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/recordingrules/delete.py @@ -35,3 +35,9 @@ def delete_rules(cmd, cluster_subscription, cluster_resource_group_name, cluster cluster_resource_group_name, "KubernetesRecordingRulesRuleGroup-{0}".format(cluster_name) ) + delete_rule( + cmd, + cluster_subscription, + cluster_resource_group_name, + truncate_rule_group_name("UXRecordingRulesRuleGroup - {0}".format(cluster_name)) + ) diff --git a/src/k8s-extension/setup.py b/src/k8s-extension/setup.py index 05b70a05016..c46e1aca890 100644 --- a/src/k8s-extension/setup.py +++ b/src/k8s-extension/setup.py @@ -33,7 +33,7 @@ # TODO: Add any additional SDK dependencies here DEPENDENCIES = [] -VERSION = "1.6.3" +VERSION = "1.6.5" with open("README.rst", "r", encoding="utf-8") as f: README = f.read() diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 00000000000..5687a0bf32d --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,9 @@ +settings.json +tmp/ +bin/* +!bin/connectedk8s-1.0.0-py3-none-any.whl +!bin/k8s_extension-0.4.0-py3-none-any.whl +!bin/k8s_extension_private-0.1.0-py3-none-any.whl +!bin/k8s_configuration-1.0.0-py3-none-any.whl +!bin/connectedk8s-values.yaml +*.xml \ No newline at end of file diff --git a/testing/Bootstrap.ps1 b/testing/Bootstrap.ps1 new file mode 100644 index 00000000000..7a09152ff58 --- /dev/null +++ b/testing/Bootstrap.ps1 @@ -0,0 +1,80 @@ +param ( + [switch] $SkipInstall, + [switch] $CI +) + +# Disable confirm prompt for script +az config set core.disable_confirm_prompt=true + +# Configuring the environment +$ENVCONFIG = Get-Content -Path $PSScriptRoot/settings.json | ConvertFrom-Json + +az account set --subscription $ENVCONFIG.subscriptionId + +if (-not (Test-Path -Path $PSScriptRoot/tmp)) { + New-Item -ItemType Directory -Path $PSScriptRoot/tmp +} + +if (!$SkipInstall) { + Write-Host "Removing the old connnectedk8s extension..." + az extension remove -n connectedk8s + Write-Host "Installing connectedk8s..." + az extension add -n connectedk8s + if (!$?) { + Write-Host "Unable to install connectedk8s, exiting..." + exit 1 + } +} + +Write-Host "Onboard cluster to Azure...starting!" + +az group show --name $envConfig.resourceGroup +if (!$?) { + Write-Host "Resource group does not exist, creating it now in region 'eastus2euap'" + az group create --name $envConfig.resourceGroup --location eastus2euap + + if (!$?) { + Write-Host "Failed to create Resource Group - exiting!" + Exit 1 + } +} + +# Skip creating the AKS Cluster if this is CI +if (!$CI) { + az aks show -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.aksClusterName + if (!$?) { + Write-Host "Cluster does not exist, creating it now" + az aks create -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.aksClusterName --generate-ssh-keys + } else { + Write-Host "Cluster already exists, no need to create it." + } + + Write-Host "Retrieving credentials for your AKS cluster..." + + az aks get-credentials -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.aksClusterName -f tmp/KUBECONFIG + if (!$?) + { + Write-Host "Cluster did not create successfully, exiting!" -ForegroundColor Red + Exit 1 + } + Write-Host "Successfully retrieved the AKS kubectl credentials" +} else { + Copy-Item $HOME/.kube/config -Destination $PSScriptRoot/tmp/KUBECONFIG +} + +az connectedk8s show -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.arcClusterName +if ($?) +{ + Write-Host "Cluster is already connected, no need to re-connect" + Exit 0 +} + +Write-Host "Connecting the cluster to Arc with connectedk8s..." +$Env:KUBECONFIG="$PSScriptRoot/tmp/KUBECONFIG" +az connectedk8s connect -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.arcClusterName -l uksouth +if (!$?) +{ + kubectl get pods -A + Exit 1 +} +Write-Host "Successfully onboarded the cluster to Azure" \ No newline at end of file diff --git a/testing/Cleanup.ps1 b/testing/Cleanup.ps1 new file mode 100644 index 00000000000..cef1e2d1bc3 --- /dev/null +++ b/testing/Cleanup.ps1 @@ -0,0 +1,30 @@ +param ( + [switch] $CI +) + +# Disable confirm prompt for script +az config set core.disable_confirm_prompt=true + +$ENVCONFIG = Get-Content -Path $PSScriptRoot/settings.json | ConvertFrom-Json + +az account set --subscription $ENVCONFIG.subscriptionId + +$Env:KUBECONFIG="$PSScriptRoot/tmp/KUBECONFIG" +Write-Host "Removing the connectedk8s arc agents from the cluster..." +az connectedk8s delete -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.arcClusterName +if (!$?) +{ + kubectl get pods -A + kubectl logs -l app.kubernetes.io/component=cluster-metadata-operator -n azure-arc -c cluster-metadata-operator + Exit 0 +} + +# Skip deleting the AKS Cluster if this is CI +if (!$CI) { + Write-Host "Deleting the AKS cluster from Azure..." + az aks delete -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.aksClusterName + if (Test-Path -Path $PSScriptRoot/tmp) { + Write-Host "Deleting the tmp directory from the test directory" + Remove-Item -Path $PSScriptRoot/tmp -Force -Confirm:$false + } +} \ No newline at end of file diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 00000000000..088a819a429 --- /dev/null +++ b/testing/README.md @@ -0,0 +1,128 @@ +# K8s Partner Extension Test Suite + +This repository serves as the integration testing suite for the `k8s-extension` Azure CLI module. + +## Testing Requirements + +All partners who wish to merge their __Custom Private Preview Release__ (owner: _Partner_) into the __Official Private Preview Release__ are required to author additional integration tests for their extension to ensure that their extension will continue to function correctly as more extensions are added into the __Official Private Preview Release__. + +For more information on creating these tests, see [Authoring Tests](docs/test_authoring.md) + +## Pre-Requisites + +In order to properly test all regression tests within the test suite, you must onboard an AKS cluster which you will use to generate your Azure Arc resource to test the extensions. Ensure that you have a resource group where you can onboard this cluster. + +### Required Installations + +The following installations are required in your environment for the integration tests to run correctly: + +1. [Helm 3](https://helm.sh/docs/intro/install/) +2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +3. [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) + +## Setup + +### Step 1: Install Pester + +This project contains [Pester](https://pester.dev/) test framework commands that are required for the integration tests to run. In an admin powershell terminal, run + +```powershell +Install-Module Pester -Force -SkipPublisherCheck +Import-Module Pester -PassThru +``` + +If you run into issues installing the framework, refer to the [Installation Guide](https://pester.dev/docs/introduction/installation) provided by the Pester docs. + +### Step 2: Get Test suite files + +You can either clone this repo (preferred option, since you will be adding your tests to this suite) or copy the files in this repo locally. Rest of the instructions here assume your working directory is k8spartner-extension-testing. + +### Step 3: Generate `k8s-extension` .whl package + +You can perform the generation of the `k8s-extension` .whl package by running the following command + +```bash +azdev setup -r . -e k8s-extension +azdev extension build k8s-extension +``` + +Additional guidance for build the extension .whl package can be found [here](https://github.com/Azure/azure-cli/blob/master/doc/extensions/authoring.md#building) + +### Step 4: Update the `k8s-extension`/`k8s-extension-private` .whl package + +This integration test suite references the .whl packages found in the `\bin` directory. After generating your `k8s-extension`/`k8s-extension-private` .whl package, copy your updated package into the `\bin` directory. + + +### Step 5: Create a `settings.json` + +To onboard the AKS and Arc clusters correctly, you will need to create a `settings.json` configuration. Create a new `settings.json` file by copying the contents of the `settings.template.json` into this file. Update the subscription id, resource group, and AKS and Arc cluster name fields with your specific values. + +### Step 6: Update the extension version value in `settings.json` + +To ensure that the tests point to your `k8s-extension-private` `.whl` package, change the value of the `k8s-extension-private` to match your package versioning in the format (Major.Minor.Patch.Extension). For example, the `k8s_extension_private-0.1.0.openservicemesh_5-py3-none-any.whl` whl package would have extension versions set to +```json +{ + "k8s-extension": "0.1.0", + "k8s-extension-private": "0.1.0.openservicemesh_5", + "connectedk8s": "0.3.5" +} + +``` + +_Note: Updates to the `connectedk8s` version and `k8s-extension` version can also be made by adding a different version of the `connectedk8s` and `k8s-extension` whl packages and changing the `connectedk8s` and `k8s-extension` values to match the (Major.Minor.Patch) version format shown above_ + +### Step 7: Run the Bootstrap Command +To bootstrap the environment with AKS and Arc clusters, run +```powershell +.\Bootstrap.ps1 +``` +This script will provision the AKS and Arc clusters needed to run the integration test suite + +## Testing + +### Testing All Extension Suites +To test all extension test suites, you must call `.\Test.ps1` with the `-ExtensionType` parameter set to either `Public` or `Private`. Based on this flag, the test suite will install the extension type specified below + +| `-ExtensionType` | Installs `az extension` | +| ---------------- | --------------------- | +| `Public` | `k8s-extension` | +| `Private` | `k8s-extension-private` | + +For example, when calling +```bash +.\Test.ps1 -ExtensionType Public +``` +the script will install your `k8s-extension` whl package and run the full test suite of `*.Tests.ps1` files included in the `\test\extensions` directory + +### Testing Public Extensions Only +If you only want to run the test cases against public-preview or GA extension test cases, you can use the `-OnlyPublicTests` flag to specify this +```bash +.\Test.ps1 -ExtensionType Public -OnlyPublicTests +``` + +### Testing Specific Extension Suite + +If you only want to run the test script on your specific test file, you can do so by specifying path to your extension test suite in the execution call + +```powershell +.\Test.ps1 -Path +``` +For example to call the `AzureMonitor.Tests.ps1` test suite, we run +```powershell +.\Test.ps1 -ExtensionType Public -Path .\test\extensions\public\AzureMonitor.Tests.ps1 +``` + +### Skipping Extension Re-Install + +By default the `Test.ps1` script will uninstall any old versions of `k8s-extension`/'`k8s-extension-private` and re-install the version specified in `settings.json`. If you do not want this re-installation to occur, you can specify the `-SkipInstall` flag to skip this process. + +```powershell +.\Test.ps1 -ExtensionType Public -SkipInstall +``` + +## Cleanup +To cleanup the AKS and Arc clusters you have provisioned in testing, run +```powershell +.\Cleanup.ps1 +``` +This will remove the AKS and Arc clusters as well as the `\tmp` directory that were created by the bootstrapping script. \ No newline at end of file diff --git a/testing/Test.ps1 b/testing/Test.ps1 new file mode 100644 index 00000000000..c1d4f093b2e --- /dev/null +++ b/testing/Test.ps1 @@ -0,0 +1,133 @@ +param ( + [string] $Path, + [switch] $SkipInstall, + [switch] $CI, + [switch] $ParallelCI, + [switch] $OnlyPublicTests, + + [Parameter(Mandatory=$True)] + [ValidateSet('k8s-extension','k8s-configuration', 'k8s-extension-private')] + [string]$Type +) + +# Disable confirm prompt for script +# Only show errors, don't show warnings +az config set core.disable_confirm_prompt=true +az config set core.only_show_errors=true + +$ENVCONFIG = Get-Content -Path $PSScriptRoot/settings.json | ConvertFrom-Json + +az account set --subscription $ENVCONFIG.subscriptionId + +$Env:KUBECONFIG="$PSScriptRoot/tmp/KUBECONFIG" +$TestFileDirectory="$PSScriptRoot/results" + +if (-not (Test-Path -Path $TestFileDirectory)) { + New-Item -ItemType Directory -Path $TestFileDirectory +} + +if ($Type -eq 'k8s-extension') { + $k8sExtensionVersion = $ENVCONFIG.extensionVersion.'k8s-extension' + $Env:K8sExtensionName = "k8s-extension" + + if (!$SkipInstall) { + Write-Host "Removing the old k8s-extension extension..." + az extension remove -n k8s-extension + Write-Host "Installing k8s-extension version $k8sExtensionVersion..." + az extension add --source ./bin/k8s_extension-$k8sExtensionVersion-py3-none-any.whl + if (!$?) { + Write-Host "Unable to find k8s-extension version $k8sExtensionVersion, exiting..." + exit 1 + } + } + if ($OnlyPublicTests) { + $testFilePath = "$PSScriptRoot/test/extensions/public" + } else { + $testFilePath = "$PSScriptRoot/test/extensions" + } +} elseif ($Type -eq 'k8s-extension-private') { + $k8sExtensionPrivateVersion = $ENVCONFIG.extensionVersion.'k8s-extension-private' + $Env:K8sExtensionName = "k8s-extension-private" + + if (!$SkipInstall) { + Write-Host "Removing the old k8s-extension-private extension..." + az extension remove -n k8s-extension-private + Write-Host "Installing k8s-extension-private version $k8sExtensionPrivateVersion..." + az extension add --source ./bin/k8s_extension_private-$k8sExtensionPrivateVersion-py3-none-any.whl + if (!$?) { + Write-Host "Unable to find k8s-extension-private version $k8sExtensionPrivateVersion, exiting..." + exit 1 + } + } + if ($OnlyPublicTests) { + $testFilePath = "$PSScriptRoot/test/extensions/public" + } else { + $testFilePath = "$PSScriptRoot/test/extensions" + } +} elseif ($Type -eq 'k8s-configuration') { + $k8sConfigurationVersion = $ENVCONFIG.extensionVersion.'k8s-configuration' + if (!$SkipInstall) { + Write-Host "Removing the old k8s-configuration extension..." + az extension remove -n k8s-configuration + Write-Host "Installing k8s-configuration version $k8sConfigurationVersion..." + az extension add --source ./bin/k8s_configuration-$k8sConfigurationVersion-py3-none-any.whl + } + $testFilePaths = "$PSScriptRoot/test/configurations" +} + +if ($ParallelCI) { + # This runs the tests in parallel during the CI pipline to speed up testing + + Write-Host "Invoking Pester to run tests from '$testFilePath'..." + $testFiles = @() + foreach ($paths in $testFilePaths) + { + $temp = Get-ChildItem $paths + $testFiles += $temp + } + $resultFileNumber = 0 + foreach ($testFile in $testFiles) + { + $resultFileNumber++ + $testName = Split-Path $testFile –leaf + Start-Job -ArgumentList $testName, $testFile, $resultFileNumber, $TestFileDirectory -Name $testName -ScriptBlock { + param($name, $testFile, $resultFileNumber, $testFileDirectory) + + Write-Host "$testFile to result file #$resultFileNumber" + $testResult = Invoke-Pester $testFile -Passthru -Output Detailed + $testResult | Export-JUnitReport -Path "$testFileDirectory/$name.xml" + } + } + + do { + Write-Host ">> Still running tests @ $(Get-Date –Format "HH:mm:ss")" –ForegroundColor Blue + Get-Job | Where-Object { $_.State -eq "Running" } | Format-Table –AutoSize + Start-Sleep –Seconds 30 + } while((Get-Job | Where-Object { $_.State -eq "Running" } | Measure-Object).Count -ge 1) + + Get-Job | Wait-Job + $failedJobs = Get-Job | Where-Object { -not ($_.State -eq "Completed")} + Get-Job | Receive-Job –AutoRemoveJob –Wait –ErrorAction 'Continue' + + if ($failedJobs.Count -gt 0) { + Write-Host "Failed Jobs" –ForegroundColor Red + $failedJobs + throw "One or more tests failed" + } +} elseif ($CI) { + if ($Path) { + $testFilePath = "$PSScriptRoot/$Path" + } + Write-Host "Invoking Pester to run tests from '$testFilePath'..." + $testResult = Invoke-Pester $testFilePath -Passthru -Output Detailed + $testName = Split-Path $testFilePath –leaf + $testResult | Export-JUnitReport -Path "$testFileDirectory/$testName.xml" +} else { + if ($Path) { + Write-Host "Invoking Pester to run tests from '$PSScriptRoot/$Path'" + Invoke-Pester -Output Detailed $PSScriptRoot/$Path + } else { + Write-Host "Invoking Pester to run tests from '$testFilePath'..." + Invoke-Pester -Output Detailed $testFilePath + } +} diff --git a/testing/bin/k8s_configuration-1.0.0-py3-none-any.whl b/testing/bin/k8s_configuration-1.0.0-py3-none-any.whl new file mode 100644 index 00000000000..cc8e8e0995f Binary files /dev/null and b/testing/bin/k8s_configuration-1.0.0-py3-none-any.whl differ diff --git a/testing/bin/k8s_extension-0.4.0-py3-none-any.whl b/testing/bin/k8s_extension-0.4.0-py3-none-any.whl new file mode 100644 index 00000000000..6c5113f7d11 Binary files /dev/null and b/testing/bin/k8s_extension-0.4.0-py3-none-any.whl differ diff --git a/testing/docs/test_authoring.md b/testing/docs/test_authoring.md new file mode 100644 index 00000000000..b41286ba66d --- /dev/null +++ b/testing/docs/test_authoring.md @@ -0,0 +1,142 @@ +# Test Authoring + +All partners are _required_ to author additional integration tests when merging their extension into the __Official Private Preview Release__. The information below outlines how to setup and author these additional tests. + +## Requirements + +All partners are required to cover standard CLI scenarios in your extensions testing suite. When adding these tests and preparing to merge your updated extension whl package, your tests along with the other tests in the test suite must pass at 100%. + +Standard CLI scenarios include: + +1. `az k8s-extension create` +2. `az k8s-extension show` +3. `az k8s-extension list` +4. `az k8s-extension update` +5. `az k8s-extension delete` + +In addition to these standard scenarios, if there are any rigorous parameter validation standards, these should also be included in this test suite. + +## Setup + +The setup process for test authoring is the same as setup for generic testing. See [Setup](../README.md#setup) for guidance. + +## Writing Tests + +This section outlines the common flow for creating and running additional extension integration tests for the `k8s-extension` package. + +The suite utilizes the [Pester](https://pester.dev/) framework. For more information on creating generic Pester tests, see the [Create a Pester Test](https://pester.dev/docs/quick-start#creating-a-pester-test) section in the Pester docs. + +### Step 1: Create Test File + +To create an integration test suite for your extension, create an extension test file in the format `.Tests.ps1` and place the file in one of the following directories +| Extension Type | Directory | +| ---------------------- | ----------------------------------- | +| General Availability | .\test\extensions\public | +| Public Preview | .\test\extensions\public | +| Private Preview | .\test\extensions\private-preview | + +For example, to create a test suite file for the Azure Monitor extension, I create the file `AzureMonitor.Tests.ps1` in the `\test\extensions\public` directory because Container Insights extension is in _Public Preview_. + +### Step 2: Setup Global Variables + +All test suite files must have the following structure for importing the environment config and declaring globals + +```powershell +Describe ' Testing' { + BeforeAll { + $extensionType = "" + $extensionName = "" + $extensionAgentName = "" + $extensionAgentNamespace = "" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } +} +``` + +You can declare additional global variables for your tests by adding additional powershell variable to this `BeforeAll` block. + +_Note: Commonly used constants used by all extension test suites are stored in the `Constants.ps1` file_ + +### Step 3: Add Tests + +Adding tests to the test suite can now be performed by adding `It` blocks to the outer `Describe` block. For instance to test create on a extension in the case of AzureMonitor, I write the following test: + +```powershell +Describe 'Azure Monitor Testing' { + BeforeAll { + $extensionType = "microsoft.azuremonitor.containers" + $extensionName = "azuremonitor-containers" + $extensionAgentName = "omsagent" + $extensionAgentNamespace = "kube-system" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + $output = az k8s-extension create -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters --extension-type $extensionType -n $extensionName + $? | Should -BeTrue + + $output = az k8s-extension show -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do + { + if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { + if (Get-PodStatus $extensionAgentName -Namespace $extensionAgentNamespace -eq $POD_RUNNING) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } +} +``` + +The above test calls `az k8s-extension create` to create the `azuremonitor-containers` extension and retries checking that the extension resource was actually created on the Arc cluster and that the extension status successfully returns `$SUCCESS_MESSAGE` which is equivalent to `Successfully installed the extension`. + +## Tips/Notes + +### Accessing Extension Data + +`.\Test.ps1` assumes that the user has `kubectl` and `az` installed in their environment; therefore, tests are able to access information on the extension at the service and on the arc cluster. For instance, in the above test, we access the `extensionconfig` CRDs on the arc cluster by calling + +```powershell +kubectl get extensionconfigs -A -o json +``` + +If we want to access the extension data on the cluster with a specific `$extensionName`, we run + +```powershell +(kubectl get extensionconfigs -A -o json).items | Where-Object { $_.metadata.name -eq $extensionName } +``` + +Because some of these commands are so common, we provide the following helper commands in the `test\Helper.ps1` file + +| Command | Description | +| ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| Get-ExtensionData | Retrieves the ExtensionConfig CRD in JSON format with `.meatadata.name` matching the `extensionName` | +| Get-ExtensionStatus | Retrieves the `.status.status` from the ExtensionConfig CRD with `.meatadata.name` matching the `extensionName` | +| Get-PodStatus -Namespace | Retrieves the `status.phase` from the first pod on the cluster with `.metadata.name` matching `extensionName` | + +### Stdout for Debugging + +To print out to the Console for debugging while writing your test cases use the `Write-Host` command. If you attempt to use the `Write-Output` command, it will not show because of the way that Pester is invoked + +```powershell +Write-Host "Some example output" +``` + +### Global Constants + +Looking at the above test, we can see that we are accessing the `ENVCONFIG` to retrieve the environment variables from the `settings.json`. All variables in the `settings.json` are accessible from the `ENVCONFIG`. The most useful ones for testing will be `ENVCONFIG.arcClusterName` and `ENVCONFIG.resourceGroup`. + diff --git a/testing/owners.txt b/testing/owners.txt new file mode 100644 index 00000000000..ead6f446410 --- /dev/null +++ b/testing/owners.txt @@ -0,0 +1,2 @@ +joinnis +nanthi \ No newline at end of file diff --git a/testing/pipeline/k8s-custom-pipelines.yml b/testing/pipeline/k8s-custom-pipelines.yml new file mode 100644 index 00000000000..c8beb79098b --- /dev/null +++ b/testing/pipeline/k8s-custom-pipelines.yml @@ -0,0 +1,212 @@ +trigger: + batch: true + branches: + include: + - k8s-extension/public + - k8s-extension/private +pr: + branches: + include: + - k8s-extension/public + - k8s-extension/private + +stages: +- stage: BuildTestPublishExtension + displayName: "Build, Test, and Publish Extension" + variables: + TEST_PATH: $(Agent.BuildDirectory)/s/testing + CLI_REPO_PATH: $(Agent.BuildDirectory)/s + SUBSCRIPTION_ID: "15c06b1b-01d6-407b-bb21-740b8617dea3" + RESOURCE_GROUP: "K8sPartnerExtensionTest" + BASE_CLUSTER_NAME: "k8s-extension-cluster" + IS_PRIVATE_BRANCH: $[or(eq(variables['Build.SourceBranch'], 'refs/heads/k8s-extension/private'), eq(variables['System.PullRequest.TargetBranch'], 'k8s-extension/private'))] + jobs: + - template: ./templates/run-test.yml + parameters: + jobName: AzureDefender + path: ./test/extensions/public/AzureDefender.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzureMLKubernetes + path: ./test/extensions/public/AzureMLKubernetes.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzureMonitorMetrics + path: ./test/extensions/public/AzureMonitorMetrics.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzureMonitor + path: ./test/extensions/public/AzureMonitor.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: AzurePolicy + path: ./test/extensions/public/AzurePolicy.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: ExtensionTypes + path: ./test/extensions/public/ExtensionTypes.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: OpenServiceMesh + path: ./test/extensions/public/OpenServiceMesh.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: Flux + path: ./test/extensions/public/Flux.Tests.ps1 + - job: BuildPublishExtension + pool: + vmImage: 'ubuntu-20.04' + displayName: "Build and Publish the Extension Artifact" + variables: + CLI_REPO_PATH: $(Agent.BuildDirectory)/s + workingDirectory: $(CLI_REPO_PATH) + displayName: "Setup and Build Extension with azdev" + steps: + - template: ./templates/build-extension.yml + parameters: + CLI_REPO_PATH: $(CLI_REPO_PATH) + IS_PRIVATE_BRANCH: $(IS_PRIVATE_BRANCH) + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(CLI_REPO_PATH)/dist + +- stage: AzureCLIOfficial + displayName: "Azure Official CLI Code Checks" + dependsOn: [] + jobs: + - job: CheckLicenseHeader + displayName: "Check License" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - bash: | + set -ev + + # prepare and activate virtualenv + python -m venv env/ + + chmod +x ./env/bin/activate + source ./env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install -q azdev + + azdev setup -c ../azure-cli -r ./ + + azdev --version + az --version + + azdev verify license + + - job: StaticAnalysis + displayName: "Static Analysis" + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests + displayName: 'Install wheel, pylint, flake8, requests' + + - job: IndexVerify + displayName: "Verify Extensions Index" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - bash: | + #!/usr/bin/env bash + set -ev + pip install wheel==0.30.0 requests packaging + export CI="ADO" + python ./scripts/ci/test_index.py -v + displayName: "Verify Extensions Index" + + - job: SourceTests + displayName: "Integration Tests, Build Tests" + pool: + vmImage: 'ubuntu-latest' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + Python311: + python.version: '3.11' + Python312: + python.version: '3.12' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install wheel==0.30.0 + displayName: 'Install wheel==0.30.0' + - bash: | + set -ev + + # prepare and activate virtualenv + pip install virtualenv + python -m virtualenv venv/ + source ./venv/bin/activate + + # clone azure-cli + git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install azdev + + azdev --version + + azdev setup -c ../azure-cli -r ./ -e k8s-extension + azdev test k8s-extension + displayName: 'Run integration test and build test' + + - job: LintModifiedExtensions + displayName: "CLI Linter on Modified Extensions" + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - bash: | + set -ev + + # prepare and activate virtualenv + pip install virtualenv + python -m virtualenv venv/ + source ./venv/bin/activate + + # clone azure-cli + git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install azdev + + azdev --version + + azdev setup -c ../azure-cli -r ./ -e k8s-extension + + # overwrite the default AZURE_EXTENSION_DIR set by ADO + AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version + + AZURE_EXTENSION_DIR=~/.azure/cliextensions azdev linter --include-whl-extensions k8s-extension + displayName: "CLI Linter on Modified Extension" + env: + ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) diff --git a/testing/pipeline/templates/build-extension.yml b/testing/pipeline/templates/build-extension.yml new file mode 100644 index 00000000000..6ccd671f266 --- /dev/null +++ b/testing/pipeline/templates/build-extension.yml @@ -0,0 +1,52 @@ +parameters: + CLI_REPO_PATH: "" +steps: +- bash: | + echo "Using the private preview of k8s-extension to build..." + + cp ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private -r + mv ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private/azext_k8s_extension ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private/azext_k8s_extension_private + cp ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private/setup_private.py ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private/setup.py + cp ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private/azext_k8s_extension_private/consts_private.py ${{ parameters.CLI_REPO_PATH }}/src/k8s-extension-private/azext_k8s_extension_private/consts.py + + EXTENSION_NAME="k8s-extension-private" + EXTENSION_FILE_NAME="k8s_extension_private" + + echo "##vso[task.setvariable variable=EXTENSION_NAME]$EXTENSION_NAME" + echo "##vso[task.setvariable variable=EXTENSION_FILE_NAME]$EXTENSION_FILE_NAME" + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'True')) + displayName: "Copy Files, Set Variables for k8s-extension-private" +- bash: | + echo "Using the public version of k8s-extension to build..." + + EXTENSION_NAME="k8s-extension" + EXTENSION_FILE_NAME="k8s_extension" + + echo "##vso[task.setvariable variable=EXTENSION_NAME]$EXTENSION_NAME" + echo "##vso[task.setvariable variable=EXTENSION_FILE_NAME]$EXTENSION_FILE_NAME" + condition: and(succeeded(), eq(variables['IS_PRIVATE_BRANCH'], 'False')) + displayName: "Copy Files, Set Variables for k8s-extension" +- task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 +- bash: | + set -ev + echo "Building extension ${EXTENSION_NAME}..." + + # prepare and activate virtualenv + pip install virtualenv + python3 -m venv env/ + source env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install --upgrade pip + pip install -q azdev + + ls ${{ parameters.CLI_REPO_PATH }} + + azdev --version + azdev setup -c ../azure-cli -r ${{ parameters.CLI_REPO_PATH }} -e $(EXTENSION_NAME) + azdev extension build $(EXTENSION_NAME) diff --git a/testing/pipeline/templates/run-test.yml b/testing/pipeline/templates/run-test.yml new file mode 100644 index 00000000000..526f2eba0f4 --- /dev/null +++ b/testing/pipeline/templates/run-test.yml @@ -0,0 +1,104 @@ +parameters: + jobName: '' + path: '' + +jobs: +- job: ${{ parameters.jobName}} + pool: + vmImage: 'ubuntu-20.04' + variables: + ${{ if eq(variables['IS_PRIVATE_BRANCH'], 'False') }}: + EXTENSION_NAME: "k8s-extension" + EXTENSION_FILE_NAME: "k8s_extension" + ${{ if ne(variables['IS_PRIVATE_BRANCH'], 'False') }}: + EXTENSION_NAME: "k8s-extension-private" + EXTENSION_FILE_NAME: "k8s_extension_private" + steps: + - bash: | + echo "Installing helm3" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh --version v3.6.3 + + echo "Installing kubectl" + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + kubectl version --client + displayName: "Setup the VM with helm3 and kubectl" + - template: ./build-extension.yml + parameters: + CLI_REPO_PATH: $(CLI_REPO_PATH) + - bash: | + K8S_EXTENSION_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) + echo "##vso[task.setvariable variable=K8S_EXTENSION_VERSION]$K8S_EXTENSION_VERSION" + cp * $(TEST_PATH)/bin + workingDirectory: $(CLI_REPO_PATH)/dist + displayName: "Copy the Built .whl to Extension Test Path" + + - bash: | + RAND_STR=$RANDOM + AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" + ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" + + JSON_STRING=$(jq -n \ + --arg SUB_ID "$SUBSCRIPTION_ID" \ + --arg RG "$RESOURCE_GROUP" \ + --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ + --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ + --arg K8S_EXTENSION_VERSION "$K8S_EXTENSION_VERSION" \ + '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-extension": $K8S_EXTENSION_VERSION, "k8s-extension-private": $K8S_EXTENSION_VERSION, connectedk8s: "1.0.0"}}') + echo $JSON_STRING > settings.json + cat settings.json + workingDirectory: $(TEST_PATH) + displayName: "Generate a settings.json file" + + - bash : | + echo "Downloading the kind script" + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster + displayName: "Create and Start the Kind cluster" + + - bash: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + displayName: "Upgrade az to latest version" + + - task: AzureCLI@2 + displayName: Bootstrap + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Bootstrap.ps1 -CI + workingDirectory: $(TEST_PATH) + + - task: AzureCLI@2 + displayName: Run the Test Suite for ${{ parameters.path }} + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Test.ps1 -CI -Path ${{ parameters.path }} -Type $(EXTENSION_NAME) + workingDirectory: $(TEST_PATH) + continueOnError: true + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/testing/results/*.xml' + failTaskOnFailedTests: true + condition: succeededOrFailed() + + - task: AzureCLI@2 + displayName: Cleanup + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Cleanup.ps1 -CI + workingDirectory: $(TEST_PATH) + condition: succeededOrFailed() diff --git a/testing/settings.template.json b/testing/settings.template.json new file mode 100644 index 00000000000..5129dbd0a20 --- /dev/null +++ b/testing/settings.template.json @@ -0,0 +1,11 @@ +{ + "subscriptionId": "", + "resourceGroup": "", + "aksClusterName": "", + "arcClusterName": "", + + "extensionVersion": { + "k8s-extension": "0.3.0", + "k8s-extension-private": "0.1.0" + } +} \ No newline at end of file diff --git a/testing/test/configurations/Configuration.HTTPS.Tests.ps1 b/testing/test/configurations/Configuration.HTTPS.Tests.ps1 new file mode 100644 index 00000000000..a2dee2b348f --- /dev/null +++ b/testing/test/configurations/Configuration.HTTPS.Tests.ps1 @@ -0,0 +1,54 @@ +Describe 'Source Control Configuration (HTTPS) Testing' { + BeforeAll { + $configurationName = "https-config" + . $PSScriptRoot/Constants.ps1 + . $PSScriptRoot/Helper.ps1 + + $dummyValue = "dummyValue" + $secretName = "git-auth-$configurationName" + } + + It 'Creates a configuration with https user and https key on the cluster' { + $output = az k8s-configuration create -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -u "https://github.com/Azure/arc-k8s-demo" -n $configurationName --scope cluster --https-user $dummyValue --https-key $dummyValue --operator-namespace $configurationName + $? | Should -BeTrue + + # Loop and retry until the configuration installs and helm pod comes up + $n = 0 + do + { + if (Get-ConfigStatus $configurationName -eq $SUCCESS_MESSAGE) { + if (Get-PodStatus $configurationName -Namespace $configurationName -eq $POD_RUNNING ) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + + Secret-Exists $secretName -Namespace $configurationName + } + + It "Lists the configurations on the cluster" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $? | Should -BeTrue + + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + $configExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the configuration from the cluster" { + az k8s-configuration delete -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + # Configuration should be removed from the resource model + az k8s-configuration show -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeFalse + } + + It "Performs another list after the delete" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + $configExists | Should -BeNullOrEmpty + } +} \ No newline at end of file diff --git a/testing/test/configurations/Configuration.HelmOperator.Tests.ps1 b/testing/test/configurations/Configuration.HelmOperator.Tests.ps1 new file mode 100644 index 00000000000..8b89ba24c58 --- /dev/null +++ b/testing/test/configurations/Configuration.HelmOperator.Tests.ps1 @@ -0,0 +1,137 @@ +Describe 'Source Control Configuration (Helm Operator Properties) Testing' { + BeforeAll { + $configurationName = "helm-enabled-config" + . $PSScriptRoot/Constants.ps1 + . $PSScriptRoot/Helper.ps1 + + $customOperatorParams = "--set helm.versions=v3 --set mycustomhelmvalue=yay" + $customChartVersion = "0.6.0" + } + + It 'Creates a configuration with helm enabled on the cluster' { + $output = az k8s-configuration create -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -u "https://github.com/Azure/arc-k8s-demo" -n $configurationName --scope cluster --enable-helm-operator --operator-namespace $configurationName --helm-operator-params "--set helm.versions=v3" + $? | Should -BeTrue + + # Loop and retry until the configuration installs and helm pod comes up + $n = 0 + do + { + if (Get-ConfigStatus $configurationName -eq $SUCCESS_MESSAGE) { + if (Get-PodStatus "$configurationName-helm" -Namespace $configurationName -eq $POD_RUNNING ) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Updates the helm operator params and performs a show" { + Set-ItResult -Skipped -Because "Update is not a valid scenario for now" + + az k8s-configuration update -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -n $configurationName --helm-operator-params $customOperatorParams + $? | Should -BeTrue + + $output = az k8s-configuration show --cluster-name $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + $configData = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + ($configData.helmOperatorProperties.chartValues -eq $customOperatorParams) | Should -BeTrue + + # Loop and retry until the configuration updates + $n = 0 + do + { + $helmOperatorChartValues = (Get-ConfigData $configurationName).spec.helmOperatorProperties.chartValues + if ($helmOperatorChartValues -ne $null -And $helmOperatorChartValues.ToString() -eq $customOperatorParams) { + if (Get-ConfigStatus $configurationName -Match $SUCCESS_MESSAGE) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Updates the helm operator chart version and performs a show" { + Set-ItResult -Skipped -Because "Update is not a valid scenario for now" + + az k8s-configuration update -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -n $configurationName --helm-operator-chart-version $customChartVersion + $? | Should -BeTrue + + $output = az k8s-configuration show --cluster-name $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + # Check that the helmOperatorProperties chartValues didn't change + $configData = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + ($configData.helmOperatorProperties.chartValues -eq $customOperatorParams) | Should -BeTrue + ($configData.helmOperatorProperties.chartVersion -eq $customChartVersion) | Should -BeTrue + + # Loop and retry until the configuration updates + $n = 0 + do + { + $helmOperatorChartVersion = (Get-ConfigData $configurationName).spec.helmOperatorProperties.chartVersion + if ($helmOperatorChartVersion -ne $null -And $helmOperatorChartVersion.ToString() -eq $customChartVersion) { + if (Get-ConfigStatus $configurationName -Match $SUCCESS_MESSAGE) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Disables the helm operator on the cluster" { + Set-ItResult -Skipped -Because "Update is not a valid scenario for now" + + az k8s-configuration update -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -n $configurationName --enable-helm-operator=false + $? | Should -BeTrue + + $output = az k8s-configuration show --cluster-name $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + $helmOperatorEnabled = ($output | ConvertFrom-Json).enableHelmOperator + $helmOperatorEnabled.ToString() -eq "False" | Should -BeTrue + + # Loop and retry until the configuration updates + $n = 0 + do { + $helmOperatorEnabled = (Get-ConfigData $configurationName).spec.enableHelmOperator + if ($helmOperatorEnabled -ne $null -And $helmOperatorEnabled.ToString() -eq "False") { + if (Get-ConfigStatus $configurationName -Match $SUCCESS_MESSAGE) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Lists the configurations on the cluster" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $? | Should -BeTrue + + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + $configExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the configuration from the cluster" { + az k8s-configuration delete -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + # Configuration should be removed from the resource model + az k8s-configuration show -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeFalse + } + + It "Performs another list after the delete" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + $configExists | Should -BeNullOrEmpty + } +} \ No newline at end of file diff --git a/testing/test/configurations/Configuration.KnownHost.Tests.ps1 b/testing/test/configurations/Configuration.KnownHost.Tests.ps1 new file mode 100644 index 00000000000..2cb2946bc3e --- /dev/null +++ b/testing/test/configurations/Configuration.KnownHost.Tests.ps1 @@ -0,0 +1,6 @@ +Describe 'Source Control Configuration (SSH Configs) Testing' { + BeforeAll { + . $PSScriptRoot/Constants.ps1 + . $PSScriptRoot/Helper.ps1 + } +} \ No newline at end of file diff --git a/testing/test/configurations/Configuration.PrivateKey.Tests.ps1 b/testing/test/configurations/Configuration.PrivateKey.Tests.ps1 new file mode 100644 index 00000000000..4bf86d52012 --- /dev/null +++ b/testing/test/configurations/Configuration.PrivateKey.Tests.ps1 @@ -0,0 +1,86 @@ +Describe 'Source Control Configuration (SSH Configs) Testing' { + BeforeAll { + . $PSScriptRoot/Constants.ps1 + . $PSScriptRoot/Helper.ps1 + + $RSA_KEYPATH = "$TMP_DIRECTORY\rsa.private" + $DSA_KEYPATH = "$TMP_DIRECTORY\dsa.private" + $ECDSA_KEYPATH = "$TMP_DIRECTORY\ecdsa.private" + $ED25519_KEYPATH = "$TMP_DIRECTORY\ed25519.private" + + $KEY_ARR = [System.Tuple]::Create("rsa", $RSA_KEYPATH), [System.Tuple]::Create("dsa", $DSA_KEYPATH), [System.Tuple]::Create("ecdsa", $ECDSA_KEYPATH), [System.Tuple]::Create("ed25519", $ED25519_KEYPATH) + foreach ($keyTuple in $KEY_ARR) { + # Automattically say yes to overwrite with ssh-keygen + Write-Output "y" | ssh-keygen -t $keyTuple.Item1 -f $keyTuple.Item2 -P """" + } + + $SSH_GIT_URL = "git://github.com/anubhav929/flux-get-started.git" + $HTTP_GIT_URL = "https://github.com/Azure/arc-k8s-demo" + + $configDataRSA = [System.Tuple]::Create("rsa-config", $RSA_KEYPATH) + $configDataDSA = [System.Tuple]::Create("dsa-config", $DSA_KEYPATH) + $configDataECDSA = [System.Tuple]::Create("ecdsa-config", $ECDSA_KEYPATH) + $configDataED25519 = [System.Tuple]::Create("ed25519-config", $ED25519_KEYPATH) + + $CONFIG_ARR = $configDataRSA, $configDataDSA, $configDataECDSA, $configDataED25519 + } + + It 'Creates a configuration with each type of ssh private key' { + foreach($configData in $CONFIG_ARR) { + az k8s-configuration create -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -u $SSH_GIT_URL -n $configData.Item1 --scope cluster --operator-namespace $configData.Item1 --ssh-private-key-file $configData.Item2 + $? | Should -BeTrue + } + + # Loop and retry until the configuration installs and helm pod comes up + $n = 0 + do + { + $readyConfigs = 0 + foreach($configData in $CONFIG_ARR) { + # TODO: Change this to checking the success message after we merge in the bugfix into the agent + if (Get-PodStatus $configData.Item1 -Namespace $configData.Item1 -eq $POD_RUNNING) { + $readyConfigs += 1 + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le 30 -And $readyConfigs -ne 4) + $n | Should -BeLessOrEqual 30 + } + + It 'Fails when trying to create a configuration with ssh url and https auth values' { + az k8s-configuration create -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -u $HTTP_GIT_URL -n "config-should-fail" --scope cluster --operator-namespace "config-should-fail" --ssh-private-key-file $RSA_KEYPATH + $? | Should -BeFalse + } + + It "Lists the configurations on the cluster" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $? | Should -BeTrue + + foreach ($configData in $CONFIG_ARR) { + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configData.Item1 } + $configExists | Should -Not -BeNullOrEmpty + } + } + + It "Deletes the configuration from the cluster" { + foreach ($configData in $CONFIG_ARR) { + az k8s-configuration delete -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configData.Item1 + $? | Should -BeTrue + + # Configuration should be removed from the resource model + az k8s-configuration show -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configData.Item1 + $? | Should -BeFalse + } + } + + It "Performs another list after the delete" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $? | Should -BeTrue + + foreach ($configData in $CONFIG_ARR) { + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configData.Item1 } + $configExists | Should -BeNullOrEmpty + } + } +} \ No newline at end of file diff --git a/testing/test/configurations/Configuration.Tests.ps1 b/testing/test/configurations/Configuration.Tests.ps1 new file mode 100644 index 00000000000..a85df42ed2e --- /dev/null +++ b/testing/test/configurations/Configuration.Tests.ps1 @@ -0,0 +1,80 @@ +Describe 'Basic Source Control Configuration Testing' { + BeforeAll { + $configurationName = "basic-config" + . $PSScriptRoot/Constants.ps1 + . $PSScriptRoot/Helper.ps1 + } + + It 'Creates a configuration and checks that it onboards correctly' { + az k8s-configuration create -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -u "https://github.com/Azure/arc-k8s-demo" -n $configurationName --scope cluster --enable-helm-operator=false --operator-namespace $configurationName + $? | Should -BeTrue + + # Loop and retry until the configuration installs + $n = 0 + do + { + if (Get-ConfigStatus $configurationName -Match $SUCCESS_MESSAGE) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the configuration" { + $output = az k8s-configuration show -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type "connectedClusters" -n $configurationName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Runs an update on the configuration on the cluster" { + Set-ItResult -Skipped -Because "Update is not a valid scenario for now" + + az k8s-configuration update -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName --enable-helm-operator + $? | Should -BeTrue + + $output = az k8s-configuration show --cluster-name $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + $helmOperatorEnabled = ($output | ConvertFrom-Json).enableHelmOperator + $helmOperatorEnabled.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the configuration updates + $n = 0 + do { + $helmOperatorEnabled = (Get-ConfigData $configurationName).spec.enableHelmOperator + if ($helmOperatorEnabled -And $helmOperatorEnabled.ToString() -eq "True") { + if (Get-ConfigStatus $configurationName -Match $SUCCESS_MESSAGE) { + break + } + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Lists the configurations on the cluster" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $? | Should -BeTrue + + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + $configExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the configuration from the cluster" { + az k8s-configuration delete -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeTrue + + # Configuration should be removed from the resource model + az k8s-configuration show -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters -n $configurationName + $? | Should -BeFalse + } + + It "Performs another list after the delete" { + $output = az k8s-configuration list -c $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters + $configExists = $output | ConvertFrom-Json | Where-Object { $_.id -Match $configurationName } + $configExists | Should -BeNullOrEmpty + } +} \ No newline at end of file diff --git a/testing/test/configurations/Constants.ps1 b/testing/test/configurations/Constants.ps1 new file mode 100644 index 00000000000..f1e8c6ffdc3 --- /dev/null +++ b/testing/test/configurations/Constants.ps1 @@ -0,0 +1,8 @@ +$ENVCONFIG = Get-Content -Path $PSScriptRoot\..\..\settings.json | ConvertFrom-Json +$SUCCESS_MESSAGE = "Successfully installed the operator" +$FAILED_MESSAGE = "Failed the install of the operator" +$TMP_DIRECTORY = "$PSScriptRoot\..\..\tmp" + +$POD_RUNNING = "Running" + +$MAX_RETRY_ATTEMPTS = 18 \ No newline at end of file diff --git a/testing/test/configurations/Helper.ps1 b/testing/test/configurations/Helper.ps1 new file mode 100644 index 00000000000..842e2da84aa --- /dev/null +++ b/testing/test/configurations/Helper.ps1 @@ -0,0 +1,45 @@ +function Get-ConfigData { + param( + [string]$configName + ) + + $output = kubectl get gitconfigs -A -o json | ConvertFrom-Json + return $output.items | Where-Object { $_.metadata.name -eq $configurationName } +} + +function Get-ConfigStatus { + param( + [string]$configName + ) + + $configData = Get-ConfigData $configName + if ($configData -ne $null) { + return $configData.status.status + } + return $null +} + +function Get-PodStatus { + param( + [string]$podName, + [string]$Namespace + ) + + $allPodData = kubectl get pods -n $Namespace -o json | ConvertFrom-Json + $podData = $allPodData.items | Where-Object { $_.metadata.name -Match $podName } + return $podData.status.phase +} + +function Secret-Exists { + param( + [string]$secretName, + [string]$Namespace + ) + + $allSecretData = kubectl get secrets -n $Namespace -o json | ConvertFrom-Json + $secretData = $allSecretData.items | Where-Object { $_.metadata.name -Match $secretName } + if ($secretData.Length -ge 1) { + return $true + } + return $false +} \ No newline at end of file diff --git a/testing/test/extensions/data/azure_ml/test_cert.pem b/testing/test/extensions/data/azure_ml/test_cert.pem new file mode 100644 index 00000000000..a8cfe1294b4 --- /dev/null +++ b/testing/test/extensions/data/azure_ml/test_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFlzCCA3+gAwIBAgIULOv9pTNMMZNMXFF3ctTJZEcPfQgwDQYJKoZIhvcNAQEL +BQAwWzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UEAwwLdGVzdGluZi5jb20w +HhcNMjIwOTA4MDMxNDA0WhcNMjMwOTA4MDMxNDA0WjBbMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMRQwEgYDVQQDDAt0ZXN0aW5mLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAM1dfkb2+o/VzazOSSXlv03rRaV4/+wKALuOGrZGJbGzm4DJ +WyN6+zogIKIbDMPUm4eB1Jh9vSQ0RciwhCUogPel7HYQhlFtyfq4I2PReXm7MInm +ZGpkFZdHliYFMQv5O1FDN4SjHbpKpEomUIngj+j5IRuQ+5wksJfEE4GKBdTv0zSJ +kvIW3o4qf3kYUuoT53g93HqUQKg6LKZ0ra7som1F1dVuOgYXmyirrdFm2SJ6y4Z7 +Fuu7/bNHJAeUIcaw/3FZrKAdYst5MO6BquDPQ3LUmIW9MPO7ynfR0MKMXFvHmQmt +ve9IMRF52FkKqO6mut3kMEhQqE9iiEcLZDUghRwzxIgYU1cHw1jBCTxois4f6HdY +zK9fK9TZoI1ftWoFcofhV4uiB8NwsRQqFeAsMrd2qCbMjLoHysMbKl3ORwtG+sl0 +ti3DhiLQbedfBHzy0xtaZvkauP6+qoGYYjGiDQjP+acvCrjjSwVpWWhu44EWFWnc +Iszjp2AaG4LagaV+ZjTHDa4mkyz/dI3EDZ6wDmHCYdbI57yJhXqjSooVYNE4FYwo +KDXVT+42mtBGIA+e/pRFYE1bUsuIubhuKVLNd081W+rsS3Juv6KWMNnahUm8Mv1P +unwQKrtJtc31rEXGjsVp4E95ncbu/0aAoczCnYlXp1pLNevnNfRdeoOWnNLzAgMB +AAGjUzBRMB0GA1UdDgQWBBQNlOeOEqZh7hojtGoKETkk+Cdl/zAfBgNVHSMEGDAW +gBQNlOeOEqZh7hojtGoKETkk+Cdl/zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQB3Dmv2Ftxp0vTDBZMUXjOKqpyuznpT5Y7LzoHxWKBXnwCGVuOo +1P1sse/QD87No+jAi3lPtQo1+GVbnN8kkIPAu8Cu3ogrlMr9938ogWz1/x33D2Nh +DLk2ZDg7UxsGxIKvMCV9MopZGQ2HCN7M43iXJ5dpZ82F6kMHZpGdigAnbTzh8iGu +vro1SiBCwPwlv+VW7VvU0OpGgBQlVj8CMON01/lWYZd4vXrv5iox59l/b1HcNrYN +Pe2CvZAmqx4Wnar8HLV6TAFPqvFf/6kAeWXt89mKaGx/LTy632sXeBHVnL62o6OD +WUjLECjTF1LSI/tzgoQUKIJ70FysIayUoSDcyb+jpb0dKBoi1vUQOS3R5gH35Z0c +fiGXvsDCtK8UvU5V4W8msBtEF1TB1vk5rrhAIGecHyY87iQnF/Lue6CvLQ6jhga6 +A7RRaKf05Stz+pyG3AF3P6iAyFK72k9LIpb+iGvlDZ2yWAMcqEuTRGeo3G3xQabN +VGG0thneWrQ7KicRXNavMgytnHs73mHGTMgffg4njcNPuio8ewcwxhDycYJpr4gz +NqtdgFPBzx9nQjrK6ZBDSzbkkLFNLnr+OSB9pvlpVnC957qqZ1L0GrJIfN/kbCF3 +TG7u3bkbTjlrvuxY1FAqrXesZrlBpx5cxLrO8XOuSlFB0q2pJSwfIv6Jbw== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/testing/test/extensions/data/azure_ml/test_key.pem b/testing/test/extensions/data/azure_ml/test_key.pem new file mode 100644 index 00000000000..e238b5c2225 --- /dev/null +++ b/testing/test/extensions/data/azure_ml/test_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDNXX5G9vqP1c2s +zkkl5b9N60WleP/sCgC7jhq2RiWxs5uAyVsjevs6ICCiGwzD1JuHgdSYfb0kNEXI +sIQlKID3pex2EIZRbcn6uCNj0Xl5uzCJ5mRqZBWXR5YmBTEL+TtRQzeEox26SqRK +JlCJ4I/o+SEbkPucJLCXxBOBigXU79M0iZLyFt6OKn95GFLqE+d4Pdx6lECoOiym +dK2u7KJtRdXVbjoGF5soq63RZtkiesuGexbru/2zRyQHlCHGsP9xWaygHWLLeTDu +gargz0Ny1JiFvTDzu8p30dDCjFxbx5kJrb3vSDERedhZCqjuprrd5DBIUKhPYohH +C2Q1IIUcM8SIGFNXB8NYwQk8aIrOH+h3WMyvXyvU2aCNX7VqBXKH4VeLogfDcLEU +KhXgLDK3dqgmzIy6B8rDGypdzkcLRvrJdLYtw4Yi0G3nXwR88tMbWmb5Grj+vqqB +mGIxog0Iz/mnLwq440sFaVlobuOBFhVp3CLM46dgGhuC2oGlfmY0xw2uJpMs/3SN +xA2esA5hwmHWyOe8iYV6o0qKFWDROBWMKCg11U/uNprQRiAPnv6URWBNW1LLiLm4 +bilSzXdPNVvq7Etybr+iljDZ2oVJvDL9T7p8ECq7SbXN9axFxo7FaeBPeZ3G7v9G +gKHMwp2JV6daSzXr5zX0XXqDlpzS8wIDAQABAoICAGA91WT6b7gikW3PitY40it4 ++72tc/oxQeCjmv8a5qVdr51uP8jj5IJ79e8iUBwiMfUSMgh4vMAPwzhnCLbFQZNN +bgByhA/7LLHTw7oOvCgBQqENmLeHSdsIkGQnALJEzbiqkIUXUGIygsXBKPNEiwy6 +W/qoOlIVm7C0EhQeE9eTwN4ZLwVHFGt5nR2p+Yl7ZHmkPAQyIA72nGAxxAd7HC+r +j6ejLYwXWf54XlAJK+8Nrv3KB5bYFfADge4PTLjpz/xV8yFiRB9pHzZXDDaoy0ow +OX5LiHpg4mS+rl/OGaZlZuHzS1Ss91niSTKJXVviRSahvsLVEduKKKVqwD5pjBcx +fRQOFIVX6Hk8aizHZaF01nZ48Y8Lyi/i6ZmajKxBODuVXhJXz6IWxREsUPNzJvHH +MSwnN8Mx3zx45dzVlgm8lDv56o0OWI4I2ppRs2dowWWbItjVHWdpreWxp2uMNYQB +o9lId5XEKisRhI0WRd42/CNUaYKf1ikAvw3d2UhrhteGQotYBaXVdhoH4g5z1F+A +wAfZXEif5gQoL3shHtQtdB9WSZhCiY7DUQhIKt2sqXOPEFwJUTnwyCrQymKUNKBj +BEtM1jDUT2ganQgNxGE9OtThFnrosnP95rSzCj87mrne56ZtMPNAmHiZt9JN4mfM +C4JDu22nqJKQ5FZMewwBAoIBAQDnCibtimNnl+SEhGK5SbRLgyRAY+AbwzSU4gWa +YyisIUmFb7YlwkjmiSSH0Bt5HUfgJssHtgUmOwHMGVEL8PpyNohzfP3Fe7yfJhxA +mIWZLYnvsnSRB4eVuUtk93ot0XaFJAbzMzOkuSWkWnlIf4Rdiv5JGwNiEwlOujGy +snzUOsFzT0j2lGzHIBaZYPC/L25kp8JQ7lrrsQOoF103lrO6Uqn0KTLGfRNYf6ZU +1FmOGkyIsl+csrI1lOyVDjTVi4XFgCv8EUeWaGM2jLaEIWmAYfzbmJamf0rNeF1H +Mh5Y5iSkOjzmuz31NGu+cW7MV4IFzYuoRC/MZbFX6KOabYDzAoIBAQDjjUPGmVh6 +eJQpSUB2KbJgj86DAilMLwpmdMrmyPNbZaiA0R9x5w7YWPbh4K48OFPnzJsntB7I +hNdniTq9qUPt5cgtDdHJ1PvdZ/DkPn8hOig/KpyPcUvkG3femtv3J1cNclqbX4ID +LoytnmHT2wB9V0frgp+G7JPAM75BlJwkgxhUnkyxn+4yPuoxasDYneh2bZUgxzYw +PcWKBM/j4gbIolg332ItsMcsKK/a0HwD2JWEYkkePrdcWE2T9qfacAdaoZaaZS4R +D2LU3eMmRotPKXIuI3JPYJmVWSZZndn+ysgmmYzen7/d9p8G1ZrA2N1dYDoVf3jh +X6A9N3YJl+YBAoIBAHjff9RA1ZbKCb0mwbusis4C00F4vzPnIahOw52tCQdc9uj/ +s+z3Q0qRL3J6dxUbM5Ja2Ve0a+c/ccZE7Hjx3yVH0IWTO/VIsjsVJizJXwPvpj2o +QIHrzYyQf5hYPSyhbH9lhNlRzU/9qWreBpveUvLZmAXJQzDZQsJUeVHDPbmO78yT +C1ot9ucKq6gc5ncvqnKwreHHgfvTBVW4u4Usq+TsAIyDzVO49hkT14KEAkJtEeNm +Zs1FVCTiQBAPeabLMvZMAzcCF1DiVh2g6pAgJuEK4s5Ee3SqHgl3Ul3AI85gwYTG +Dzyrc1PI1CGzmMMBeT3t9oXW/qbSAUE7rfRKG+8CggEAJGtCkrGOSKOtyuHPcFoC +E5RQkAUziN7qgjVlGATHdjRSALP3nWpGpPewI7yrBjZZr3q+xl78okkolIiRHzPN +DHE/VX6lufDdkrUFB/K8tBuzv1BZmFegttRyne0ZEXh5ZUyNFdr2Wv4DQ/JaY+bk +MCtc9mOElrqcdyGQ7LwVNX7J0Rk42yDmpaIOJ3SXgtPbFcE6IfHgSV5JlGpqv2U4 +groA9ohJFVj6t6WXZ6UAhDkQzQxR+YY+IIh9ehX7DWnqs2WzTeits8tLnRgaN9EI +kNXoUVwY+n1Sd2W6TpOGBVJ9MDhZJHRa5/KFxzk+uGi9HSm+ghxRw3hjlAihWq22 +AQKCAQEAgOtb7QKUIyhQh4sqAflyEzZyIvJ8+luM7ZaeXevBgarXBCOlrcfvsv4r +qdFOIV9ZFfBkf40dSwlHu50DiPnZSg8aRBXjRVlWP9QkabnB5hPwjZ7Rr3+F44ty +gr76n6yJbiLZIBOAlDs0e9W30vcereqvr6puO8TOZbjBm+41gyDnw0MzQODQFqgg +CngrENoMGjsBPAsRzO2Q0NPlUQx5o+qbc+izMdRj532jPZ4rymlfVAFLQ5qBJcVT +A09HY7HJu2GvCfoBRUICD1etd3tg/m9hK4rSrnLn+zLRjfg1v4FPxVK2ZO2+XY0G +hsfzUDNwvb4vnu9Yq3m//0pk/pMs6g== +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/testing/test/extensions/public/AzureDefender.Tests.ps1 b/testing/test/extensions/public/AzureDefender.Tests.ps1 new file mode 100644 index 00000000000..419d226cb2e --- /dev/null +++ b/testing/test/extensions/public/AzureDefender.Tests.ps1 @@ -0,0 +1,71 @@ +Describe 'Azure Defender Testing' { + BeforeAll { + $extensionType = "microsoft.azuredefender.kubernetes" + $extensionName = "microsoft.azuredefender.kubernetes" + $extensionAgentNamespace = "azuredefender" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + $output = az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do + { + # Only check the extension config, not the pod since this doesn't bring up pods + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $provisioningState = ($output | ConvertFrom-Json).provisioningState + Write-Host "Got ProvisioningState: $provisioningState for the extension" + if (Has-ExtensionData $extensionName) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } +} diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 new file mode 100644 index 00000000000..d38627219c1 --- /dev/null +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -0,0 +1,221 @@ +Describe 'AzureML Kubernetes Testing' { + BeforeAll { + $extensionType = "Microsoft.AzureML.Kubernetes" + $extensionName = "azureml-kubernetes-connector" + $extensionAgentNamespace = "azureml" + $relayResourceIDKey = "relayserver.hybridConnectionResourceID" + $serviceBusResourceIDKey = "servicebus.resourceID" + $mockUpdateKey = "mockTest" + $mockProtectedUpdateKey = "mockProtectedTest" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { + $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" + $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train stable --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceRouterServiceType=nodePort sslCname=testinf.com --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile --no-wait + $? | Should -BeTrue + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do + { + if (Has-ExtensionData $extensionName) { + break + } + Start-Sleep -Seconds 20 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + + # check if relay is populated + $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + $relayResourceID | Should -Not -BeNullOrEmpty + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Wait for the extension to be ready" { + # Loop and retry until the extension installed + $n = 0 + do + { + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $provisioningState = ($output | ConvertFrom-Json).provisioningState + Write-Host "Provisioning state: $provisioningState" + if ($provisioningState -eq "Succeeded") { + break + } + Start-Sleep -Seconds 20 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Perform Update extension" { + $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" + $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" + az $Env:K8sExtensionName update -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --config "$($mockUpdateKey)=true" --config-protected "$($mockProtectedUpdateKey)=true" sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile --no-wait + $? | Should -BeTrue + + # Loop and retry until the extension updated + $n = 0 + do + { + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $provisioningState = ($output | ConvertFrom-Json).provisioningState + Write-Host "Provisioning state: $provisioningState" + if ($provisioningState -eq "Succeeded") { + break + } + Start-Sleep -Seconds 20 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + + $mockedUpdateData = Get-ExtensionConfigurationSettings $extensionName $mockUpdateKey + $mockedUpdateData | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster with inference enabled" { + # cleanup the relay and servicebus + $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + $relayNamespaceName = $relayResourceID.split("/")[8] + az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName + + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } + + # It 'Creates the extension and checks that it onboards correctly with training enabled' { + # az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true + # $? | Should -BeTrue + + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue + + # $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + # $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # # Loop and retry until the extension installs + # $n = 0 + # do + # { + # if (Has-ExtensionData $extensionName) { + # break + # } + # Start-Sleep -Seconds 20 + # $n += 1 + # } while ($n -le $MAX_RETRY_ATTEMPTS) + # $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + + # # check if relay is populated + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $relayResourceID | Should -Not -BeNullOrEmpty + # } + + # It "Deletes the extension from the cluster" { + # # cleanup the relay and servicebus + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey + # $relayNamespaceName = $relayResourceID.split("/")[8] + # $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] + # az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName + # az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName + + # $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue + + # # Extension should not be found on the cluster + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeFalse + # $output | Should -BeNullOrEmpty + # } + + # It 'Creates the extension and checks that it onboards correctly with inference enabled' { + # az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false + # $? | Should -BeTrue + + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue + + # $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + # $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # # Loop and retry until the extension installs + # $n = 0 + # do + # { + # if (Has-ExtensionData $extensionName) { + # break + # } + # Start-Sleep -Seconds 20 + # $n += 1 + # } while ($n -le $MAX_RETRY_ATTEMPTS) + # $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + + # # check if relay is populated + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $relayResourceID | Should -Not -BeNullOrEmpty + # } + + # It "Deletes the extension from the cluster with inference enabled" { + # # cleanup the relay and servicebus + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey + # $relayNamespaceName = $relayResourceID.split("/")[8] + # $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] + # az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName + # az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName + + # $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeTrue + + # # Extension should not be found on the cluster + # $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + # $? | Should -BeFalse + # $output | Should -BeNullOrEmpty + # } +} diff --git a/testing/test/extensions/public/AzureMonitor.Tests.ps1 b/testing/test/extensions/public/AzureMonitor.Tests.ps1 new file mode 100644 index 00000000000..9748755eea0 --- /dev/null +++ b/testing/test/extensions/public/AzureMonitor.Tests.ps1 @@ -0,0 +1,68 @@ +Describe 'Azure Monitor Testing' { + BeforeAll { + $extensionType = "microsoft.azuremonitor.containers" + $extensionName = "azuremonitor-containers" + $extensionAgentName = "omsagent" + $extensionAgentNamespace = "kube-system" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do + { + if (Has-ExtensionData $extensionName) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } +} diff --git a/testing/test/extensions/public/AzureMonitorMetrics.Tests.ps1 b/testing/test/extensions/public/AzureMonitorMetrics.Tests.ps1 new file mode 100644 index 00000000000..d1eca16ddb8 --- /dev/null +++ b/testing/test/extensions/public/AzureMonitorMetrics.Tests.ps1 @@ -0,0 +1,69 @@ +Describe 'Azure Monitor Metrics Testing' { + Describe 'Azure Monitor Metrics Testing' { + BeforeAll { + $extensionType = "microsoft.azuremonitor.containers.metrics" + $extensionName = "azuremonitor-metrics" + $extensionAgentName = "ama-metrics" + $extensionAgentNamespace = "kube-system" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do { + if (Has-ExtensionData $extensionName) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } + } +} diff --git a/testing/test/extensions/public/AzurePolicy.Tests.ps1 b/testing/test/extensions/public/AzurePolicy.Tests.ps1 new file mode 100644 index 00000000000..8f68426f4ed --- /dev/null +++ b/testing/test/extensions/public/AzurePolicy.Tests.ps1 @@ -0,0 +1,74 @@ +Describe 'Azure Policy Testing' { + BeforeAll { + $extensionType = "microsoft.policyinsights" + $extensionName = "policy" + $extensionAgentName = "azure-policy" + $extensionAgentNamespace = "kube-system" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --no-wait + $? | Should -BeTrue + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # Check that we get the principal id back for the created identity + $principalId = ($output | ConvertFrom-Json).identity.principalId + $principalId | Should -Not -BeNullOrEmpty + + # Loop and retry until the extension installs + $n = 0 + do + { + # Only check the extension config, not the pod since this doesn't bring up pods + $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + if (Has-ExtensionData $extensionName){ + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } +} diff --git a/testing/test/extensions/public/Dapr.Tests.ps1 b/testing/test/extensions/public/Dapr.Tests.ps1 new file mode 100644 index 00000000000..5af40546c19 --- /dev/null +++ b/testing/test/extensions/public/Dapr.Tests.ps1 @@ -0,0 +1,63 @@ +Describe 'DAPR Testing' { + BeforeAll { + $extensionType = "microsoft.dapr" + $extensionName = "dapr" + $clusterType = "connectedClusters" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + $output = az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName --extension-type $extensionType --configuration-settings "skipExistingDaprCheck=true" --no-wait + $? | Should -BeTrue + + $n = 0 + do + { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName + $provisioningState = ($output | ConvertFrom-Json).provisioningState + Write-Host "Provisioning State: $provisioningState" + if ($provisioningState -eq "Succeeded") { + break + } + Start-Sleep -Seconds 40 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } +} \ No newline at end of file diff --git a/testing/test/extensions/public/ExtensionTypes.Tests.ps1 b/testing/test/extensions/public/ExtensionTypes.Tests.ps1 new file mode 100644 index 00000000000..12b6e559ccb --- /dev/null +++ b/testing/test/extensions/public/ExtensionTypes.Tests.ps1 @@ -0,0 +1,58 @@ +Describe 'Extension Types Testing' { + BeforeAll { + $extensionType = "microsoft.contoso.samples" + $location = "eastus2euap" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Performs a show-by-cluster extension types call' { + Start-Sleep -Seconds 240 + $output = az $Env:K8sExtensionName extension-types show-by-cluster -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It 'Performs a show-by-location extension types call' { + $output = az $Env:K8sExtensionName extension-types show-by-location --location $location --extension-type $extensionType + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Performs a list-by-cluster extension types call" { + $output = az $Env:K8sExtensionName extension-types list-by-cluster -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Performs a list-by-location extension types call" { + $output = az $Env:K8sExtensionName extension-types list-by-location --location $location + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Performs a show-version-by-cluster extension type version call" { + $output = az $Env:K8sExtensionName extension-types show-version-by-cluster -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType --version 1.1.0 + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Performs a show-version-by-location extension type version call" { + $output = az $Env:K8sExtensionName extension-types show-version-by-location --location $location --extension-type $extensionType --version 1.1.0 + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Performs a list-versions-by-cluster extension type versions call" { + $output = az $Env:K8sExtensionName extension-types list-versions-by-cluster -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Performs a list-versions-by-location extension type versions call" { + $output = az $Env:K8sExtensionName extension-types list-versions-by-location --location $location --extension-type $extensionType + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } +} diff --git a/testing/test/extensions/public/Flux.Tests.ps1 b/testing/test/extensions/public/Flux.Tests.ps1 new file mode 100644 index 00000000000..7faa6aa9c70 --- /dev/null +++ b/testing/test/extensions/public/Flux.Tests.ps1 @@ -0,0 +1,63 @@ +Describe 'Azure Flux Testing' { + BeforeAll { + $extensionType = "microsoft.flux" + $extensionName = "flux" + $clusterType = "connectedClusters" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + It 'Creates the extension and checks that it onboards correctly' { + $output = az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName --extension-type $extensionType --no-wait + $? | Should -BeTrue + + $n = 0 + do + { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName + $provisioningState = ($output | ConvertFrom-Json).provisioningState + Write-Host "Provisioning State: $provisioningState" + if ($provisioningState -eq "Succeeded") { + break + } + Start-Sleep -Seconds 40 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type $clusterType + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } +} \ No newline at end of file diff --git a/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 new file mode 100644 index 00000000000..07c00e4d42d --- /dev/null +++ b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 @@ -0,0 +1,72 @@ +Describe 'Azure OpenServiceMesh Testing' { + BeforeAll { + $extensionType = "microsoft.openservicemesh" + $extensionName = "openservicemesh" + $extensionVersion = "1.0.0" + $extensionAgentName = "osm-controller" + $extensionAgentNamespace = "arc-osm-system" + $releaseTrain = "pilot" + + . $PSScriptRoot/../../helper/Constants.ps1 + . $PSScriptRoot/../../helper/Helper.ps1 + } + + # Should Not BeNullOrEmpty checks if the command returns JSON output + + It 'Creates the extension and checks that it onboards correctly' { + az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train $releaseTrain --version $extensionVersion --auto-upgrade false --no-wait + $? | Should -BeTrue + + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + + $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + $isAutoUpgradeMinorVersion.ToString() -eq "False" | Should -BeTrue + + # Loop and retry until the extension installs + $n = 0 + do + { + if (Has-ExtensionData $extensionName) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Performs a show on the extension" { + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + + $output | Should -Not -BeNullOrEmpty + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType } + $extensionExists | Should -Not -BeNullOrEmpty + } + + It "Deletes the extension from the cluster" { + $output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName --force + $? | Should -BeTrue + + # Extension should not be found on the cluster + $output = az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName + $? | Should -BeFalse + $output | Should -BeNullOrEmpty + } + + It "Performs another list after the delete" { + $output = az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters + $? | Should -BeTrue + $output | Should -Not -BeNullOrEmpty + + $extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionName } + $extensionExists | Should -BeNullOrEmpty + } +} diff --git a/testing/test/helper/Constants.ps1 b/testing/test/helper/Constants.ps1 new file mode 100644 index 00000000000..f93b10e1c11 --- /dev/null +++ b/testing/test/helper/Constants.ps1 @@ -0,0 +1,7 @@ +$ENVCONFIG = Get-Content -Path $PSScriptRoot/../../settings.json | ConvertFrom-Json +$SUCCESS_MESSAGE = "Successfully installed the extension" +$FAILED_MESSAGE = "Failed to install the extension" + +$POD_RUNNING = "Running" + +$MAX_RETRY_ATTEMPTS = 15 \ No newline at end of file diff --git a/testing/test/helper/Helper.ps1 b/testing/test/helper/Helper.ps1 new file mode 100644 index 00000000000..4ff949e7ab4 --- /dev/null +++ b/testing/test/helper/Helper.ps1 @@ -0,0 +1,72 @@ +function Get-ExtensionData { + param( + [string]$extensionName + ) + + $output = kubectl get extensionconfigs -A -o json | ConvertFrom-Json + return $output.items | Where-Object { $_.metadata.name -eq $extensionName } +} + +function Has-ExtensionData { + param( + [string]$extensionName + ) + $extensionData = Get-ExtensionData $extensionName + if ($extensionData) { + return $true + } + return $false +} + + +function Has-Identity-Provisioned { + $output = kubectl get azureclusteridentityrequests -n azure-arc container-insights-clusteridentityrequest -o json | ConvertFrom-Json + return ($null -ne $output.status.expirationTime) -and ($null -ne $output.status.tokenReference.dataName) -and ($null -ne $output.status.tokenReference.secretName) +} + +function Get-ExtensionStatus { + param( + [string]$extensionName + ) + + $extensionData = Get-ExtensionData $extensionName + if ($extensionData) { + return $extensionData.status.status + } + return $null +} + +function Get-PodStatus { + param( + [string]$podName, + [string]$Namespace + ) + + $allPodData = kubectl get pods -n $Namespace -o json | ConvertFrom-Json + $podData = $allPodData.items | Where-Object { $_.metadata.name -Match $podName } + if ($podData.Length -gt 1) { + return $podData[0].status.phase + } + return $podData.status.phase +} + +function Get-ExtensionConfigurationSettings { + param( + [string]$extensionName, + [string]$configKey + ) + + $extensionData = Get-ExtensionData $extensionName + if ($extensionData) { + return $extensionData.spec.parameter."$configKey" + } + return $null +} + +function Check-Error { + param( + [string]$output + ) + $hasError = $output -CMatch "ERROR" + return $hasError +} \ No newline at end of file