Skip to content

Commit 81b6b3f

Browse files
aim-for-betterZhenyu Zhou
aim-for-better
and
Zhenyu Zhou
authored
{HDInsight} Support creating cluster with ADLSGen1/2 storage in national clouds (#12310)
* Support creating cluster with Adlsgen1/2 storage in national clouds * Update ChangeLog.md * Refactor the ResetClusterIdentity function Co-authored-by: Zhenyu Zhou <[email protected]>
1 parent 602a49e commit 81b6b3f

File tree

10 files changed

+4248
-3013
lines changed

10 files changed

+4248
-3013
lines changed

‎src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ Create cluster
5858
function Create-Cluster{
5959
param(
6060
[string] $clusterName="hdi-ps-test",
61-
[string] $location="West US",
61+
[string] $location="East US",
6262
[string] $resourceGroupName="group-ps-test",
6363
[string] $clusterType="Spark",
6464
[string] $storageAccountName="storagepstest",

‎src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightClusterTests/TestClusterRelatedCommands.json‎

Lines changed: 936 additions & 914 deletions
Large diffs are not rendered by default.

‎src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightClusterTests/TestCmkClusterRelatedCommands.json‎

Lines changed: 640 additions & 771 deletions
Large diffs are not rendered by default.

‎src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightHostTests/TestHostRelatedCommands.json‎

Lines changed: 788 additions & 650 deletions
Large diffs are not rendered by default.

‎src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightSetGatewayCredentialTest/TestSetGatewayCredential.json‎

Lines changed: 1849 additions & 663 deletions
Large diffs are not rendered by default.

‎src/HDInsight/HDInsight.Test/UnitTests/NewClusterTests.cs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ public void CanCreateNewHDInsightCluster()
112112
parameters.Location == Location &&
113113
parameters.UserName == _httpCred.UserName &&
114114
parameters.Password == _httpCred.Password.ConvertToString() &&
115-
parameters.ClusterType == ClusterType), It.IsAny<string>()))
115+
parameters.ClusterType == ClusterType), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
116116
.Returns(cluster)
117117
.Verifiable();
118118

@@ -317,7 +317,7 @@ private void CreateNewHDInsightCluster(
317317
parameters.SshUserName == _sshCred.UserName &&
318318
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
319319
((!setEdgeNodeVmSize && parameters.EdgeNodeSize == null) || (setEdgeNodeVmSize && parameters.EdgeNodeSize == "edgeNodeVmSizeSetTest")) &&
320-
(workerNodeDataDisks == 0) || (workerNodeDataDisks > 0 && parameters.WorkerNodeDataDisksGroups.First().DisksPerNode == workerNodeDataDisks)), It.IsAny<string>()))
320+
(workerNodeDataDisks == 0) || (workerNodeDataDisks > 0 && parameters.WorkerNodeDataDisksGroups.First().DisksPerNode == workerNodeDataDisks)), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
321321
.Returns(cluster)
322322
.Verifiable();
323323

@@ -416,7 +416,7 @@ public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
416416
parameters.ClusterType == sparkClusterType &&
417417
parameters.SshUserName == _sshCred.UserName &&
418418
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
419-
parameters.ComponentVersion["Spark"] == componentVersion["Spark"]), It.IsAny<string>()))
419+
parameters.ComponentVersion["Spark"] == componentVersion["Spark"]), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
420420
.Returns(cluster)
421421
.Verifiable();
422422
hdinsightManagementMock.Setup(
@@ -431,7 +431,7 @@ public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
431431
parameters.Password == _httpCred.Password.ConvertToString() &&
432432
parameters.ClusterType == ClusterType &&
433433
parameters.SshUserName == _sshCred.UserName &&
434-
parameters.SshPassword == _sshCred.Password.ConvertToString()), It.IsAny<string>()))
434+
parameters.SshPassword == _sshCred.Password.ConvertToString()), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
435435
.Returns(cluster)
436436
.Verifiable();
437437

@@ -564,7 +564,7 @@ public void CanCreateNewHDInsightCluster_Disk_Encryption()
564564
parameters.DiskEncryptionProperties.KeyName == EncryptionKeyName &&
565565
parameters.DiskEncryptionProperties.KeyVersion == EncryptionKeyVersion &&
566566
parameters.DiskEncryptionProperties.EncryptionAlgorithm == EncryptionAlgorithm &&
567-
parameters.DiskEncryptionProperties.MsiResourceId == AssignedIdentity), It.IsAny<string>()))
567+
parameters.DiskEncryptionProperties.MsiResourceId == AssignedIdentity), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
568568
.Returns(cluster)
569569
.Verifiable();
570570

@@ -604,7 +604,7 @@ public void TestStorageAccountTypeDefaultsToAzureStorage()
604604
cmdlet.DefaultStorageAccountKey = StorageKey;
605605
cmdlet.ClusterType = ClusterType;
606606

607-
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.IsAny<ClusterCreateParameters>(), It.IsAny<string>()));
607+
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.IsAny<ClusterCreateParameters>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()));
608608

609609
cmdlet.ExecuteCmdlet();
610610

@@ -617,7 +617,7 @@ public void TestStorageAccountTypeDefaultsToAzureStorage()
617617
parameters.Location == Location &&
618618
parameters.UserName == _httpCred.UserName &&
619619
parameters.Password == _httpCred.Password.ConvertToString() &&
620-
parameters.ClusterType == ClusterType), It.IsAny<string>()),
620+
parameters.ClusterType == ClusterType), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()),
621621
Times.Once);
622622
}
623623

@@ -635,7 +635,7 @@ public void TestStorageAccountTypeAzureStorage()
635635
cmdlet.ClusterType = ClusterType;
636636
cmdlet.DefaultStorageAccountType = StorageType.AzureStorage;
637637

638-
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.IsAny<ClusterCreateParameters>(), It.IsAny<string>()));
638+
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.IsAny<ClusterCreateParameters>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()));
639639

640640
cmdlet.ExecuteCmdlet();
641641

@@ -648,7 +648,7 @@ public void TestStorageAccountTypeAzureStorage()
648648
parameters.Location == Location &&
649649
parameters.UserName == _httpCred.UserName &&
650650
parameters.Password == _httpCred.Password.ConvertToString() &&
651-
parameters.ClusterType == ClusterType), It.IsAny<string>()),
651+
parameters.ClusterType == ClusterType), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()),
652652
Times.Once);
653653
}
654654

@@ -667,7 +667,7 @@ public void TestStorageAccountTypeAzureDataLakeStore()
667667
cmdlet.ClusterType = ClusterType;
668668
cmdlet.DefaultStorageAccountType = StorageType.AzureDataLakeStore;
669669

670-
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.IsAny<ClusterCreateParameters>(), It.IsAny<string>()));
670+
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.IsAny<ClusterCreateParameters>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()));
671671

672672
cmdlet.ExecuteCmdlet();
673673

@@ -680,7 +680,7 @@ public void TestStorageAccountTypeAzureDataLakeStore()
680680
parameters.Location == Location &&
681681
parameters.UserName == _httpCred.UserName &&
682682
parameters.Password == _httpCred.Password.ConvertToString() &&
683-
parameters.ClusterType == ClusterType), It.IsAny<string>()),
683+
parameters.ClusterType == ClusterType), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()),
684684
Times.Once);
685685
}
686686
}

‎src/HDInsight/HDInsight.Test/UnitTests/PremiumClusterTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public void CanCreateNewPremiumHDInsightCluster()
113113
parameters.SshUserName == _httpCred.UserName &&
114114
parameters.SshPassword == _httpCred.Password.ConvertToString() &&
115115
parameters.ClusterType == ClusterType &&
116-
parameters.ClusterTier == Tier.Premium), It.IsAny<string>()))
116+
parameters.ClusterTier == Tier.Premium),It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
117117
.Returns(cluster)
118118
.Verifiable();
119119

‎src/HDInsight/HDInsight/ChangeLog.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Supported creating cluster with ADLSGen1/2 storage in national clouds.
2122

2223
## Version 3.3.0
2324
* Supported listing hosts and restart specific hosts of the HDInsight cluster.

‎src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,7 @@ var storageAccount in
523523
};
524524
}
525525

526-
var cluster = HDInsightManagementClient.CreateNewCluster(ResourceGroupName, ClusterName, OSType, parameters, MinSupportedTlsVersion);
526+
var cluster = HDInsightManagementClient.CreateNewCluster(ResourceGroupName, ClusterName, OSType, parameters, MinSupportedTlsVersion, this.DefaultContext.Environment.ActiveDirectoryAuthority, this.DefaultContext.Environment.DataLakeEndpointResourceId);
527527

528528
if (cluster != null)
529529
{

‎src/HDInsight/HDInsight/Models/Management/AzureHdInsightManagementClient.cs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,12 @@ public AzureHdInsightManagementClient() { }
3434

3535
private IHDInsightManagementClient HdInsightManagementClient { get; set; }
3636

37-
public virtual Cluster CreateNewCluster(string resourceGroupName, string clusterName, OSType osType, ClusterCreateParameters parameters, string minSupportedTlsVersion=default(string))
37+
public virtual Cluster CreateNewCluster(string resourceGroupName, string clusterName, OSType osType, ClusterCreateParameters parameters, string minSupportedTlsVersion = default(string), string cloudAadAuthority = default(string), string cloudDataLakeAudience =default(string))
3838
{
3939
var createParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clusterName, parameters);
4040
createParams.Properties.OsType = osType;
4141
createParams.Properties.MinSupportedTlsVersion = minSupportedTlsVersion;
42+
ResetClusterIdentity(createParams, cloudAadAuthority, cloudDataLakeAudience);
4243
return HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, createParams);
4344
}
4445

@@ -225,5 +226,23 @@ public virtual void RestartHosts(string resourceGroupName, string clusterName, I
225226
{
226227
HdInsightManagementClient.VirtualMachines.RestartHosts(resourceGroupName, clusterName, hosts);
227228
}
229+
230+
private void ResetClusterIdentity(ClusterCreateParametersExtended createParams, string aadAuthority, string dataLakeAudience)
231+
{
232+
var configuation = (Dictionary<string, Dictionary<string, string>>)createParams.Properties.ClusterDefinition.Configurations;
233+
Dictionary<string, string> clusterIdentity;
234+
if(!configuation.TryGetValue("clusterIdentity", out clusterIdentity))
235+
{
236+
return;
237+
}
238+
clusterIdentity["clusterIdentity.resourceUri"]=dataLakeAudience;
239+
240+
string aadTenantIdWithUrl;
241+
clusterIdentity.TryGetValue("clusterIdentity.aadTenantId", out aadTenantIdWithUrl);
242+
243+
const string defaultPubliCloudAadAuthority= "https://login.windows.net/";
244+
string newAadTenantIdWithUrl = aadTenantIdWithUrl?.Replace(defaultPubliCloudAadAuthority, aadAuthority);
245+
clusterIdentity["clusterIdentity.aadTenantId"]=newAadTenantIdWithUrl;
246+
}
228247
}
229248
}

0 commit comments

Comments
 (0)