Skip to content

Commit 424a70d

Browse files
2258093686v-yukunli
2258093686
and
v-yukunli
authored
[HDInsight] imporve hdinsight test coverage (#28344)
Co-authored-by: Yukun Li (Beyondsoft) <[email protected]>
1 parent 94d6d17 commit 424a70d

File tree

31 files changed

+17852
-12462
lines changed

31 files changed

+17852
-12462
lines changed

‎src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,13 @@ class ClusterCommonCreateParameter{
8282
[string] $minSupportedTlsVersion
8383
[string] $virtualNetworkId
8484
[string] $subnet
85+
[string] $version
8586

8687
ClusterCommonCreateParameter([string] $clusterName, [string] $location, [string] $resourceGroupName,
8788
[string] $storageAccountResourceId, [string] $clusterType, [int] $clusterSizeInNodes,
8889
[string] $storageAccountKey, [System.Management.Automation.PSCredential] $httpCredential,
8990
[System.Management.Automation.PSCredential] $sshCredential, [string] $minSupportedTlsVersion,
90-
[string] $virtualNetworkId,[string] $subnet){
91+
[string] $virtualNetworkId,[string] $subnet,[string] $version){
9192
$this.clusterName=$clusterName
9293
$this.location=$location
9394
$this.resourceGroupName=$resourceGroupName
@@ -100,6 +101,7 @@ class ClusterCommonCreateParameter{
100101
$this.minSupportedTlsVersion=$minSupportedTlsVersion
101102
$this.virtualNetworkId=$virtualNetworkId
102103
$this.subnet=$subnet
104+
$this.version=$version
103105
}
104106
}
105107

@@ -110,29 +112,25 @@ class ClusterCommonCreateParameter{
110112
function Prepare-ClusterCreateParameter{
111113
param(
112114
[string] $clusterName="ps",
113-
[string] $location="eastus",
115+
[string] $location="East Asia",
114116
[string] $resourceGroupName="group-ps-test",
115117
[string] $storageAccountName="storagepstest",
116-
[string] $clusterType="Spark",
117-
[string] $virtualNetworkId="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/yuchen-ps-test/providers/Microsoft.Network/virtualNetworks/hdi-vn-0",
118-
[string] $subnet="default"
118+
[string] $clusterType="Hadoop",
119+
[string] $virtualNetworkId="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Network/virtualNetworks/hdi-vn-0",
120+
[string] $subnet="default",
121+
[string] $version="5.1"
119122
)
120123

121124
$clusterName=Generate-Name($clusterName)
122125
$resourceGroupName=Generate-Name($resourceGroupName)
123126

124127
$resourceGroup=New-AzResourceGroup -Name $resourceGroupName -Location $location
125128

126-
$storageAccountName=Generate-StorageAccountName($storageAccountName)
127-
128-
$storageAccount= New-AzStorageAccount -ResourceGroupName $resourceGroupName -Location $location -Name $storageAccountName -TypeString Standard_RAGRS
129-
130-
$storageAccountResourceId=$storageAccount.Id
131-
$storageAccountKey=Get-AzStorageAccountKey -ResourceGroupName $resourceGroupName -Name $storageAccountName
132-
$storageAccountKey=$storageAccountKey[0].Value
129+
$storageAccountResourceId = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Storage/storageAccounts/storagepstest"
130+
$storageAccountKey = "Sanitized"
133131

134132
$httpUser="admin"
135-
$textPassword= "YourPw!00953"
133+
$textPassword= "Sanitized"
136134
$httpPassword=ConvertTo-SecureString $textPassword -AsPlainText -Force
137135
$sshUser="sshuser"
138136
$sshPassword=$httpPassword
@@ -143,7 +141,7 @@ function Prepare-ClusterCreateParameter{
143141
$minSupportedTlsVersion="1.2"
144142
return [ClusterCommonCreateParameter]::new($clusterName, $location, $resourceGroupName, $storageAccountResourceId,
145143
$clusterType, $clusterSizeInNodes, $storageAccountKey, $httpCredential,
146-
$sshCredential, $minSupportedTlsVersion, $virtualNetworkId, $subnet)
144+
$sshCredential, $minSupportedTlsVersion, $virtualNetworkId, $subnet,$version)
147145
}
148146

149147
<#

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAzureMonitorTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public HDInsightAzureMonitorTests(ITestOutputHelper output) : base(output)
2525
{
2626
}
2727

28-
[Fact(Skip = "Sikp this")]
28+
[Fact]
2929
[Trait(Category.AcceptanceType, Category.CheckIn)]
3030
public void TestAzureMonitorRelatedCommands()
3131
{

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAzureMonitorTests.ps1‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,24 +21,15 @@ function Test-AzureMonitorRelatedCommands{
2121
# Create some resources that will be used throughout test
2222
try
2323
{
24-
$location = "East US"
25-
# prepare parameter for creating parameter
26-
$params= Prepare-ClusterCreateParameter -location $location
24+
# get cluster that will be used throughout test
25+
$cluster = Get-AzHDInsightCluster -ResourceGroupName "group-ps-test" -ClusterName "ps-test-cluster"
2726

28-
# create cluster that will be used throughout test
29-
$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
30-
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
31-
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
32-
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential -Version 5.1 `
33-
-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
3427
Assert-NotNull $cluster
3528

36-
$workspaceName = Generate-Name("workspace-ps-test")
29+
# get a Log Analytics Workspace
30+
$workspaceName = "ps-la"
3731
$resourceGroupName = $cluster.ResourceGroup
38-
39-
#create a new Log Analytics Workspace
40-
$sku = "pernode"
41-
$workspace = New-AzOperationalInsightsWorkspace -Location $location -Name $workspaceName -ResourceGroupName $resourceGroupName -Sku $sku
32+
$workspace = Get-AzOperationalInsightsWorkspace -Name $workspaceName -ResourceGroupName $resourceGroupName
4233

4334
#get workspace's primaryKey
4435
$keys = Get-AzOperationalInsightsWorkspaceSharedKey -ResourceGroupName $resourceGroupName -Name $workspace.Name

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ public HDInsightClusterTests(ITestOutputHelper output) : base(output)
2525
{
2626
}
2727

28+
[Fact]
29+
[Trait(Category.AcceptanceType, Category.CheckIn)]
30+
public void TestHDInsightJobManagementCommands()
31+
{
32+
TestRunner.RunTestScript("Test-HDInsightJobManagementCommands");
33+
}
34+
2835
[Fact]
2936
[Trait(Category.AcceptanceType, Category.CheckIn)]
3037
public void TestClusterRelatedCommands()
@@ -60,7 +67,7 @@ public void TestCreateClusterWithEncryptionAtHost()
6067
TestRunner.RunTestScript("Test-CreateClusterWithEncryptionAtHost");
6168
}
6269

63-
[Fact(Skip = "Sikp this")]
70+
[Fact]
6471
[Trait(Category.AcceptanceType, Category.CheckIn)]
6572
public void TestCreateClusterWithLoadBasedAutoscale()
6673
{
@@ -109,7 +116,7 @@ public void TestCreateClusterWithAvailabilityZones()
109116
TestRunner.RunTestScript("Test-CreateClusterWithAvailabilityZones");
110117
}
111118

112-
[Fact(Skip = "need to create resource manually")]
119+
[Fact]
113120
[Trait(Category.AcceptanceType, Category.CheckIn)]
114121
public void TestCreateClusterWithPrivateLinkConfiguration()
115122
{
@@ -136,5 +143,12 @@ public void TestUpdateClusterUserAssigned()
136143
{
137144
TestRunner.RunTestScript("Test-UpdateClusterUserAssigned");
138145
}
146+
147+
[Fact]
148+
[Trait(Category.AcceptanceType, Category.CheckIn)]
149+
public void TestManagementCluster()
150+
{
151+
TestRunner.RunTestScript("Test-ManagementCluster");
152+
}
139153
}
140154
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1‎

Lines changed: 80 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,61 @@
1212
# limitations under the License.
1313
# ----------------------------------------------------------------------------------
1414

15+
<#
16+
.SYNOPSIS
17+
Tests HDInsight job submission, monitoring, and output commands.
18+
#>
19+
20+
function Test-HDInsightJobManagementCommands{
21+
try{
22+
$clusterName = "ps-test-cluster"
23+
$resourceGroupName = "group-ps-test"
24+
$httpUser="admin"
25+
$httpPassword = ConvertTo-SecureString "Sanitized" -AsPlainText -Force
26+
$httpCredential = New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
27+
# test Use-AzHDInsightCluster
28+
Use-AzHDInsightCluster -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential
29+
30+
# test Get-AzHDInsightProperty
31+
$property = Get-AzHDInsightProperty -Location "East Asia"
32+
Assert-NotNull $property
33+
34+
# test New-AzHDInsightHiveJobDefinition
35+
$hiveJob = New-AzHDInsightHiveJobDefinition -Query "select count(*) from default.hivesampletable" -JobName "QuerySampleTable"
36+
37+
# test Start-AzHDInsightJob
38+
$jobHive = Start-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -JobDefinition $hiveJob -HttpCredential $httpCredential
39+
40+
# test Wait-AzHDInsightJob
41+
$waitJobHive = Wait-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential -JobId $jobHive.JobId
42+
Assert-NotNull $waitJobHive
43+
44+
# test Get-AzHDInsightJob
45+
$jobStatus = Get-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential -JobId $jobHive.JobId
46+
Assert-AreEqual $jobStatus.State "SUCCEEDED"
47+
48+
# test New-AzHDInsightMapReduceJobDefinition
49+
$mapReduceJob = New-AzHDInsightMapReduceJobDefinition -JarFile "/example/jars/hadoop-mapreduce-examples.jar" -ClassName "pi" -Arguments "10","10" -JobName "PiEstimation"
50+
51+
$jobMapReduce = Start-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -JobDefinition $mapReduceJob -HttpCredential $httpCredential
52+
53+
# test Stop-AzHDInsightJob
54+
Stop-AzHDInsightJob -ClusterName $clusterName -ResourceGroupName $resourceGroupName -HttpCredential $httpCredential -JobId $jobMapReduce.JobId
55+
56+
$pigJob = New-AzHDInsightPigJobDefinition -Query "SHOW TABLES"
57+
Assert-NotNull $pigJob
58+
59+
$sqoopJob = New-AzHDInsightSqoopJobDefinition
60+
Assert-NotNull $sqoopJob
61+
62+
$streamingJob = New-AzHDInsightStreamingMapReduceJobDefinition -InputPath '/tmp'
63+
Assert-NotNull $streamingJob
64+
}
65+
finally
66+
{
67+
}
68+
}
69+
1570
<#
1671
.SYNOPSIS
1772
Test Create and resize Azure HDInsight Cluster
@@ -171,7 +226,7 @@ function Test-CreateClusterWithLoadBasedAutoscale{
171226
try
172227
{
173228
# prepare parameter for creating parameter
174-
$params= Prepare-ClusterCreateParameter -location "East US"
229+
$params= Prepare-ClusterCreateParameter
175230

176231
# create autoscale cofiguration
177232
$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
@@ -181,7 +236,7 @@ function Test-CreateClusterWithLoadBasedAutoscale{
181236
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
182237
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
183238
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
184-
-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
239+
-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 5.1 `
185240
-AutoscaleConfiguration $autoscaleConfiguration -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
186241

187242
Assert-NotNull $cluster
@@ -191,7 +246,7 @@ function Test-CreateClusterWithLoadBasedAutoscale{
191246
finally
192247
{
193248
# Delete cluster and resource group
194-
# Remove-AzHDInsightCluster -ClusterName $cluster.Name
249+
Remove-AzHDInsightCluster -ClusterName $cluster.Name
195250
Remove-AzResourceGroup -ResourceGroupName $params.resourceGroupName
196251
}
197252
}
@@ -497,7 +552,7 @@ function Test-CreateClusterWithPrivateLinkConfiguration{
497552
$params= Prepare-ClusterCreateParameter
498553

499554
# Private Link requires vnet has firewall, this is difficult to create dynamically, just hardcode here
500-
$vnetId= "/subscriptions/964c10bb-8a6c-43bc-83d3-6b318c6c7305/resourceGroups/zzy-test-rg/providers/Microsoft.Network/virtualNetworks/zzytestvnet"#"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/fakevnet"
555+
$vnetId= "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Network/virtualNetworks/hdi-vn-0"
501556
$subnetName="default"
502557

503558
$ipConfigName="ipconfig"
@@ -518,7 +573,7 @@ function Test-CreateClusterWithPrivateLinkConfiguration{
518573
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
519574
-MinSupportedTlsVersion $params.minSupportedTlsVersion `
520575
-VirtualNetworkId $vnetId -SubnetName $subnetName `
521-
-ResourceProviderConnection Outbound -PrivateLink Enabled -PrivateLinkConfiguration $privateLinkConfiguration
576+
-ResourceProviderConnection Outbound -PrivateLink Enabled -PrivateLinkConfiguration $privateLinkConfiguration -Version 5.1
522577

523578
Assert-AreEqual $cluster.NetworkProperties.ResourceProviderConnection Outbound
524579
Assert-AreEqual $cluster.NetworkProperties.PrivateLink Enabled
@@ -610,3 +665,23 @@ function Test-UpdateClusterUserAssigned{
610665
# Remove-AzResourceGroup -ResourceGroupName $params.resourceGroupName
611666
}
612667
}
668+
669+
670+
<#
671+
.SYNOPSIS
672+
Test Management cluster
673+
#>
674+
function Test-ManagementCluster{
675+
try
676+
{
677+
$resourceGroup="group-ps-test"
678+
$clusterName="ps-test-cluster"
679+
$httpUser="admin"
680+
$httpPassword = ConvertTo-SecureString "Sanitized" -AsPlainText -Force
681+
$clusterCreds = New-Object System.Management.Automation.PSCredential($httpUser, $httpPassword)
682+
Set-AzHDInsightGatewayCredential -ClusterName $clusterName -ResourceGroupName $resourceGroup -HttpCredential $clusterCreds
683+
}
684+
finally
685+
{
686+
}
687+
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightConfigurationTests.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,12 @@ public void TestConfigurationPipelining()
3131
{
3232
TestRunner.RunTestScript("Test-ConfigurationPipelining");
3333
}
34+
35+
[Fact]
36+
[Trait(Category.AcceptanceType, Category.CheckIn)]
37+
public void TestCreateClusterByConfigurationPipelining()
38+
{
39+
TestRunner.RunTestScript("Test-CreateClusterByConfigurationPipelining");
40+
}
3441
}
3542
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightConfigurationTests.ps1‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,4 +96,70 @@
9696
$config = $config | Add-AzHDInsightSecurityProfile -Domain $domain -DomainUserCredential $domainUserCredential -OrganizationalUnitDN $organizationalUnitDN -LdapsUrls $ldapsUrls -ClusterUsersGroupDNs $clusterUsersGroupDNs
9797
Assert-AreEqual $config.SecurityProfile.Domain $domain
9898
Assert-NotNull $config.SecurityProfile.LdapsUrls
99-
}
99+
}
100+
101+
<#
102+
SYNOPSIS
103+
Tests create Hadoop cluster by pipelining config
104+
#>
105+
function Test-CreateClusterByConfigurationPipelining{
106+
try
107+
{
108+
# prepare parameter for creating parameter
109+
$params= Prepare-ClusterCreateParameter
110+
#test New-AzHDInsightClusterConfig
111+
$config = New-AzHDInsightClusterConfig
112+
113+
#test Set-AzHDInsightDefaultStorage
114+
$config = Set-AzHDInsightDefaultStorage -Config $config `
115+
-StorageAccountResourceId "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/group-ps-test/providers/Microsoft.Storage/storageAccounts/storagepstest" `
116+
-StorageAccountKey "Sanitized"
117+
118+
#test Add-AzHDInsightStorage
119+
$config = Add-AzHDInsightStorage -Config $config `
120+
-StorageAccountName "storagepstest.blob.core.windows.net" `
121+
-StorageAccountKey "Sanitized"
122+
123+
#test Add-AzHDInsightComponentVersion
124+
$config = Add-AzHDInsightComponentVersion -Config $config -ComponentName "Hadoop" -ComponentVersion "3.3"
125+
126+
#test Add-AzHDInsightConfigValue
127+
$config = Add-AzHDInsightConfigValue -Config $config -Core @{coreconfig = 'corevalue'}
128+
$ipConfiguration = New-AzHDInsightIPConfiguration
129+
$privateLinkConfiguration = New-AzHDInsightPrivateLinkConfiguration
130+
Assert-NotNull $ipConfiguration
131+
Assert-NotNull $privateLinkConfiguration
132+
#test Add-AzHDInsightScriptAction
133+
$config = Add-AzHDInsightScriptAction -Config $config -NodeType HeadNode -Uri "https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh" -Name "InstallHue" -Parameters "-version latest -port 20000"
134+
135+
#test Add-AzHDInsightMetastore
136+
$sqlUser="username"
137+
$sqlPassword = ConvertTo-SecureString "Sanitized" -AsPlainText -Force
138+
$sqlCredential = New-Object System.Management.Automation.PSCredential($sqlUser, $sqlPassword)
139+
$config = Add-AzHDInsightMetastore -Config $config `
140+
-MetastoreType HiveMetastore -SqlAzureServerName "pstestsqldbserver.database.windows.net" `
141+
-DatabaseName "pstestdb01" -Credential $sqlCredential
142+
143+
$clusterParams = @{
144+
ClusterType = $params.clusterType
145+
ClusterSizeInNodes = $params.clusterSizeInNodes
146+
ResourceGroupName = $params.resourceGroupName
147+
ClusterName = $params.clusterName
148+
HttpCredential = $params.httpCredential
149+
SshCredential = $params.sshCredential
150+
Location = $params.location
151+
MinSupportedTlsVersion = $params.minSupportedTlsVersion
152+
VirtualNetworkId = $params.virtualNetworkId
153+
SubnetName = $params.subnet
154+
Version = $params.version
155+
Config = $config
156+
}
157+
$cluster = New-AzHDInsightCluster @clusterParams
158+
Assert-NotNull $cluster
159+
}
160+
finally
161+
{
162+
# Delete cluster and resource group
163+
Remove-AzResourceGroup -ResourceGroupName $params.resourceGroupName
164+
}
165+
}

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightMonitoringTests.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ function Test-MonitoringRelatedCommands{
2121
# Create some resources that will be used throughout test
2222
try
2323
{
24-
$location = "EastUS"
24+
$location = "East Asia"
2525
# prepare parameter for creating parameter
2626
$params= Prepare-ClusterCreateParameter
2727

@@ -30,7 +30,7 @@ function Test-MonitoringRelatedCommands{
3030
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
3131
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3232
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
33-
-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
33+
-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default" -Version $params.version
3434

3535
$workspaceName = Generate-Name("workspace-ps-test")
3636
$resourceGroupName = $cluster.ResourceGroup

‎src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightScriptActionTests.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ function Test-ScriptActionRelatedCommands{
2929
-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
3030
-StorageAccountResourceId $params.storageAccountResourceId -StorageAccountKey $params.storageAccountKey `
3131
-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
32-
-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default"
32+
-MinSupportedTlsVersion $params.minSupportedTlsVersion -VirtualNetworkId $params.virtualNetworkId -SubnetName "default" -Version $params.version
3333

3434
$scriptActionName = Generate-Name("scriptaction")
3535
$uri = "https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh"

0 commit comments

Comments
 (0)