[HDInsight] imporve hdinsight test coverage

src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1

@@ -82,12 +82,13 @@ class ClusterCommonCreateParameter{
       [string] $minSupportedTlsVersion
       [string] $virtualNetworkId
       [string] $subnet
+      [string] $version
 
 	  ClusterCommonCreateParameter([string] $clusterName, [string] $location, [string] $resourceGroupName,
                                    [string] $storageAccountResourceId, [string] $clusterType, [int] $clusterSizeInNodes, 
                                    [string] $storageAccountKey, [System.Management.Automation.PSCredential] $httpCredential,
                                    [System.Management.Automation.PSCredential] $sshCredential, [string] $minSupportedTlsVersion,
-                                   [string] $virtualNetworkId,[string] $subnet){
+                                   [string] $virtualNetworkId,[string] $subnet,[string] $version){
                 $this.clusterName=$clusterName
                 $this.location=$location
                 $this.resourceGroupName=$resourceGroupName
@@ -100,6 +101,7 @@ class ClusterCommonCreateParameter{
                 $this.minSupportedTlsVersion=$minSupportedTlsVersion
                 $this.virtualNetworkId=$virtualNetworkId
                 $this.subnet=$subnet
+                $this.version=$version
       }
 }
 
@@ -110,26 +112,22 @@ class ClusterCommonCreateParameter{
 function Prepare-ClusterCreateParameter{
     param(
       [string] $clusterName="ps",
-      [string] $location="eastus",
+      [string] $location="East Asia",
       [string] $resourceGroupName="group-ps-test",
 	  [string] $storageAccountName="storagepstest",
-      [string] $clusterType="Spark",
-      [string] $virtualNetworkId="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/yuchen-ps-test/providers/Microsoft.Network/virtualNetworks/hdi-vn-0",
-      [string] $subnet="default"
+      [string] $clusterType="Hadoop",
+      [string] $virtualNetworkId="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Network/virtualNetworks/hdi-vn-0",
+      [string] $subnet="default",
+      [string] $version="5.1"
     )
 
     $clusterName=Generate-Name($clusterName)
     $resourceGroupName=Generate-Name($resourceGroupName)
 
     $resourceGroup=New-AzResourceGroup -Name $resourceGroupName -Location $location
 
-    $storageAccountName=Generate-StorageAccountName($storageAccountName)
-
-	$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString Standard_RAGRS
-
-	$storageAccountResourceId=$storageAccount.Id
-    $storageAccountKey=Get-AzStorageAccountKey -ResourceGroupName $resourceGroupName -Name $storageAccountName
-    $storageAccountKey=$storageAccountKey[0].Value
+    $storageAccountResourceId = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Storage/storageAccounts/storagepstest"
+    $storageAccountKey = "Sanitized"
 
     $httpUser="admin"
     $textPassword= "YourPw!00953"
@@ -143,7 +141,7 @@ function Prepare-ClusterCreateParameter{
     $minSupportedTlsVersion="1.2"
     return [ClusterCommonCreateParameter]::new($clusterName, $location,  $resourceGroupName, $storageAccountResourceId, 
                                                $clusterType, $clusterSizeInNodes, $storageAccountKey, $httpCredential,
-                                               $sshCredential, $minSupportedTlsVersion, $virtualNetworkId, $subnet)
+                                               $sshCredential, $minSupportedTlsVersion, $virtualNetworkId, $subnet,$version)
 }
 
 <#

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAzureMonitorTests.cs

@@ -25,7 +25,7 @@ public HDInsightAzureMonitorTests(ITestOutputHelper output) : base(output)
         {
         }
 
-        [Fact(Skip = "Sikp this")]
+        [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAzureMonitorRelatedCommands()
         {

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAzureMonitorTests.ps1

@@ -21,24 +21,15 @@ function Test-AzureMonitorRelatedCommands{
 	# Create some resources that will be used throughout test 
 	try
 	{
-		$location = "East US"
-		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameter -location $location
+		# get cluster that will be used throughout test
+		$cluster = Get-AzHDInsightCluster -ResourceGroupName  "group-ps-test" -ClusterName  "ps-test-cluster"
 
-		# create cluster that will be used throughout test
-		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
-		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
-		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
-		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential -Version 5.1 `
-		-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
 		Assert-NotNull $cluster
 
-		$workspaceName = Generate-Name("workspace-ps-test")
+		# get a Log Analytics Workspace
+		$workspaceName = "ps-la"
 		$resourceGroupName = $cluster.ResourceGroup
-
-		#create a new Log Analytics Workspace
-		$sku = "pernode"
-		$workspace = New-AzOperationalInsightsWorkspace -Location $location -Name $workspaceName -ResourceGroupName $resourceGroupName -Sku $sku
+		$workspace = Get-AzOperationalInsightsWorkspace -Name $workspaceName -ResourceGroupName $resourceGroupName 
 
 		#get workspace's primaryKey
 		$keys = Get-AzOperationalInsightsWorkspaceSharedKey -ResourceGroupName $resourceGroupName -Name $workspace.Name

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs

@@ -25,6 +25,13 @@ public HDInsightClusterTests(ITestOutputHelper output) : base(output)
         {
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestHDInsightJobManagementCommands()
+        {
+            TestRunner.RunTestScript("Test-HDInsightJobManagementCommands");
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestClusterRelatedCommands()
@@ -60,7 +67,7 @@ public void TestCreateClusterWithEncryptionAtHost()
             TestRunner.RunTestScript("Test-CreateClusterWithEncryptionAtHost");
         }
 
-        [Fact(Skip = "Sikp this")]
+        [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestCreateClusterWithLoadBasedAutoscale()
         {
@@ -136,5 +143,12 @@ public void TestUpdateClusterUserAssigned()
         {
             TestRunner.RunTestScript("Test-UpdateClusterUserAssigned");
         }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestManagementCluster()
+        {
+            TestRunner.RunTestScript("Test-ManagementCluster");
+        }
     }
 }

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1

@@ -12,6 +12,61 @@
 # limitations under the License.
 # ----------------------------------------------------------------------------------
 
+<#
+.SYNOPSIS
+Tests HDInsight job submission, monitoring, and output commands.
+#>
+
+function Test-HDInsightJobManagementCommands{
+	try{
+		$clusterName = "ps-test-cluster" 
+		$resourceGroupName = "group-ps-test"
+		$httpUser="admin"
+		$httpPassword = ConvertTo-SecureString "Password1234!" -AsPlainText -Force
+		$httpCredential = New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
+		# test Use-AzHDInsightCluster
+		Use-AzHDInsightCluster -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential
+
+		# test Get-AzHDInsightProperty
+		$property = Get-AzHDInsightProperty  -Location "East Asia"
+		Assert-NotNull $property
+
+		# test New-AzHDInsightHiveJobDefinition
+		$hiveJob = New-AzHDInsightHiveJobDefinition -Query "select count(*) from default.hivesampletable" -JobName "QuerySampleTable"
+
+		# test Start-AzHDInsightJob
+		$jobHive = Start-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -JobDefinition $hiveJob -HttpCredential $httpCredential
+
+		# test Wait-AzHDInsightJob
+		$waitJobHive = Wait-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential -JobId  $jobHive.JobId
+		Assert-NotNull $waitJobHive
+
+		# test Get-AzHDInsightJob
+		$jobStatus = Get-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential -JobId $jobHive.JobId
+		Assert-AreEqual $jobStatus.State "SUCCEEDED"
+
+		# test New-AzHDInsightMapReduceJobDefinition
+		$mapReduceJob = New-AzHDInsightMapReduceJobDefinition -JarFile "/example/jars/hadoop-mapreduce-examples.jar" -ClassName "pi" -Arguments "10","10" -JobName "PiEstimation"
+
+		$jobMapReduce = Start-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -JobDefinition $mapReduceJob -HttpCredential $httpCredential
+
+		# test Stop-AzHDInsightJob
+		Stop-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential -JobId  $jobMapReduce.JobId
+
+		$pigJob = New-AzHDInsightPigJobDefinition -Query "SHOW TABLES"
+		Assert-NotNull $pigJob
+
+		$sqoopJob = New-AzHDInsightSqoopJobDefinition
+		Assert-NotNull $sqoopJob
+
+		$streamingJob = New-AzHDInsightStreamingMapReduceJobDefinition -InputPath '/tmp'
+		Assert-NotNull $streamingJob
+	}
+	finally
+	{
+	}
+}
+
 <#
 .SYNOPSIS
 Test Create and resize Azure HDInsight Cluster
@@ -171,7 +226,7 @@ function Test-CreateClusterWithLoadBasedAutoscale{
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameter -location "East US"
+		$params= Prepare-ClusterCreateParameter
 
 		# create autoscale cofiguration
 		$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
@@ -181,7 +236,7 @@ function Test-CreateClusterWithLoadBasedAutoscale{
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
 		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
-		-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 5.1 `
 		-AutoscaleConfiguration $autoscaleConfiguration -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
 
 		Assert-NotNull $cluster
@@ -191,7 +246,7 @@ function Test-CreateClusterWithLoadBasedAutoscale{
 	finally
 	{
 		# Delete cluster and resource group
-		# Remove-AzHDInsightCluster -ClusterName $cluster.Name
+		Remove-AzHDInsightCluster -ClusterName $cluster.Name
 		Remove-AzResourceGroup -ResourceGroupName $params.resourceGroupName
 	}
 }
@@ -610,3 +665,23 @@ function Test-UpdateClusterUserAssigned{
 		# Remove-AzResourceGroup -ResourceGroupName $params.resourceGroupName
 	}
 }
+
+
+<#
+.SYNOPSIS
+Test Management cluster 
+#>
+function Test-ManagementCluster{
+	try
+	{
+		$resourceGroup="group-ps-test"
+		$clusterName="ps-test-cluster"
+		$httpUser="admin"
+		$httpPassword = ConvertTo-SecureString "Password1234!" -AsPlainText -Force
+		$clusterCreds = New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
+		Set-AzHDInsightGatewayCredential -ClusterName $clusterName -ResourceGroupName $resourceGroup -HttpCredential $clusterCreds
+ 	}
+	finally
+	{
+	}
+}

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightConfigurationTests.cs

@@ -31,5 +31,12 @@ public void TestConfigurationPipelining()
         {
             TestRunner.RunTestScript("Test-ConfigurationPipelining");
         }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestCreateClusterByConfigurationPipelining()
+        {
+            TestRunner.RunTestScript("Test-CreateClusterByConfigurationPipelining");
+        }
     }
 }

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightConfigurationTests.ps1

@@ -96,4 +96,70 @@
 		$config = $config | Add-AzHDInsightSecurityProfile -Domain $domain -DomainUserCredential $domainUserCredential -OrganizationalUnitDN $organizationalUnitDN -LdapsUrls $ldapsUrls -ClusterUsersGroupDNs $clusterUsersGroupDNs 
 		Assert-AreEqual $config.SecurityProfile.Domain $domain
 		Assert-NotNull $config.SecurityProfile.LdapsUrls
-    }
+    }
+
+<#
+    SYNOPSIS
+    Tests create Hadoop cluster by pipelining config 
+#>
+function Test-CreateClusterByConfigurationPipelining{
+	try
+	{
+		# prepare parameter for creating parameter
+		$params= Prepare-ClusterCreateParameter
+		#test New-AzHDInsightClusterConfig
+		$config = New-AzHDInsightClusterConfig
+
+		#test Set-AzHDInsightDefaultStorage
+		$config = Set-AzHDInsightDefaultStorage -Config $config `
+			-StorageAccountResourceId "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Storage/storageAccounts/storagepstest" `
+			-StorageAccountKey "Sanitized"
+
+		#test Add-AzHDInsightStorage
+		$config = Add-AzHDInsightStorage -Config $config `
+			-StorageAccountName "storagepstest.blob.core.windows.net" `
+			-StorageAccountKey "Sanitized"
+
+		#test Add-AzHDInsightComponentVersion
+		$config = Add-AzHDInsightComponentVersion -Config $config -ComponentName "Hadoop" -ComponentVersion "3.3"
+
+		#test Add-AzHDInsightConfigValue
+		$config = Add-AzHDInsightConfigValue -Config $config -Core @{coreconfig = 'corevalue'}
+		$ipConfiguration = New-AzHDInsightIPConfiguration
+		$privateLinkConfiguration = New-AzHDInsightPrivateLinkConfiguration
+		Assert-NotNull $ipConfiguration
+		Assert-NotNull $privateLinkConfiguration
+		#test Add-AzHDInsightScriptAction
+		$config = Add-AzHDInsightScriptAction -Config $config -NodeType HeadNode -Uri "https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh" -Name "InstallHue" -Parameters "-version latest -port 20000"
+
+		#test Add-AzHDInsightMetastore
+		$sqlUser="username"
+		$sqlPassword = ConvertTo-SecureString "Password1234!" -AsPlainText -Force
+		$sqlCredential = New-Object System.Management.Automation.PSCredential($sqlUser, $sqlPassword)
+		$config = Add-AzHDInsightMetastore -Config $config `
+			-MetastoreType HiveMetastore -SqlAzureServerName "pstestsqldbserver.database.windows.net" `
+			-DatabaseName "pstestdb01" -Credential $sqlCredential
+
+		$clusterParams = @{
+			ClusterType                     = $params.clusterType
+			ClusterSizeInNodes              = $params.clusterSizeInNodes
+			ResourceGroupName               = $params.resourceGroupName
+			ClusterName                     = $params.clusterName
+			HttpCredential                  = $params.httpCredential
+			SshCredential                   = $params.sshCredential
+			Location                        = $params.location
+			MinSupportedTlsVersion          = $params.minSupportedTlsVersion
+			VirtualNetworkId                = $params.virtualNetworkId
+			SubnetName                      = $params.subnet
+			Version                         = $params.version
+			Config                          = $config
+         }
+		$cluster = New-AzHDInsightCluster @clusterParams
+		Assert-NotNull $cluster
+	}
+	finally
+	{
+		# Delete cluster and resource group
+		Remove-AzResourceGroup -ResourceGroupName $params.resourceGroupName
+	}
+}

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightMonitoringTests.ps1

@@ -21,7 +21,7 @@ function Test-MonitoringRelatedCommands{
 	# Create some resources that will be used throughout test 
 	try
 	{
-		$location = "EastUS"
+		$location = "East Asia"
 		# prepare parameter for creating parameter
 		$params= Prepare-ClusterCreateParameter
 
@@ -30,7 +30,7 @@ function Test-MonitoringRelatedCommands{
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
 		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
-		-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default" -Version $params.version
 
 		$workspaceName = Generate-Name("workspace-ps-test")
 		$resourceGroupName = $cluster.ResourceGroup

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightScriptActionTests.ps1

@@ -29,7 +29,7 @@ function Test-ScriptActionRelatedCommands{
 		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
 		-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
 		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
-		-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default" -Version $params.version
 
 		$scriptActionName = Generate-Name("scriptaction")
 		$uri = "https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh"