Skip to content

Commit 4355b90

Browse files
authored
Update the validate set of NodeSize for spark pool related cmdlets (#20917)
* spark pool isolate compute/nodesize related changes * update change log * updateazsynapselinkconnectionlandingzonecredential
1 parent 194e4c0 commit 4355b90

17 files changed

+135
-45
lines changed

src/Synapse/Synapse/ChangeLog.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
-->
2020

2121
## Upcoming Release
22+
* Upgraded Azure.Analytics.Synapse.Artifacts to 1.0.0-preview.17
23+
* Updated `New-AzSynapseSparkPool` and `Update-AzSynapseSparkPool` to support for setting spark pool isolated compute by `-EnableIsolatedCompute`
24+
* Updated `New-AzSynapseSparkPool` and `Update-AzSynapseSparkPool` to support for setting spark pool node size to 'XLarge', 'XXLarge', 'XXXLarge'
2225

2326
## Version 2.2.0
2427
* Added breaking change message for `-SparkConfigFilePath`. It will be deprecated around the middle of December.

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/Notebooks/SetAzureSynapseNotebook.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,8 @@ public class SetAzureSynapseNotebook : SynapseArtifactsCmdletBase
7272
Mandatory = false, HelpMessage = HelpMessages.ExecutorSize)]
7373
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = SetByObjectAndSparkPool,
7474
Mandatory = false, HelpMessage = HelpMessages.ExecutorSize)]
75-
[ValidateSet(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, IgnoreCase = true)]
76-
[PSArgumentCompleter(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large)]
75+
[ValidateSet(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, Management.Synapse.Models.NodeSize.XLarge, Management.Synapse.Models.NodeSize.XXLarge, Management.Synapse.Models.NodeSize.XXXLarge, IgnoreCase = true)]
76+
[PSArgumentCompleter(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, Management.Synapse.Models.NodeSize.XLarge, Management.Synapse.Models.NodeSize.XXLarge, Management.Synapse.Models.NodeSize.XXXLarge)]
7777
public string ExecutorSize { get; set; } = Management.Synapse.Models.NodeSize.Small;
7878

7979
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = SetByNameAndSparkPool,

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkJob/SubmitAzureSynapseSparkJob.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ public class SubmitAzureSynapseSparkJob : SynapseSparkCmdletBase
112112
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RunSparkJobByParentObjectParameterSet,
113113
Mandatory = true, HelpMessage = HelpMessages.ExecutorSize)]
114114
[ValidateNotNullOrEmpty]
115-
[ValidateSet(NodeSize.Small, NodeSize.Medium, NodeSize.Large, IgnoreCase = true)]
116-
[PSArgumentCompleter(NodeSize.Small, NodeSize.Medium, NodeSize.Large)]
115+
[ValidateSet(NodeSize.Small, NodeSize.Medium, NodeSize.Large, NodeSize.XLarge, NodeSize.XXLarge, NodeSize.XXXLarge, IgnoreCase = true)]
116+
[PSArgumentCompleter(NodeSize.Small, NodeSize.Medium, NodeSize.Large, NodeSize.XLarge, NodeSize.XXLarge, NodeSize.XXXLarge)]
117117
public string ExecutorSize { get; set; }
118118

119119
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RunSparkJobParameterSetName,

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkSession/StartAzureSynapseSparkSession.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ public class StartAzureSynapseSparkSession : SynapseSparkCmdletBase
7474

7575
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.ExecutorSize)]
7676
[ValidateNotNullOrEmpty]
77-
[ValidateSet(NodeSize.Small, NodeSize.Medium, NodeSize.Large, IgnoreCase = true)]
78-
[PSArgumentCompleter(NodeSize.Small, NodeSize.Medium, NodeSize.Large)]
77+
[ValidateSet(NodeSize.Small, NodeSize.Medium, NodeSize.Large, NodeSize.XLarge, NodeSize.XXLarge, NodeSize.XXXLarge, IgnoreCase = true)]
78+
[PSArgumentCompleter(NodeSize.Small, NodeSize.Medium, NodeSize.Large, NodeSize.XLarge, NodeSize.XXLarge, NodeSize.XXXLarge)]
7979
public string ExecutorSize { get; set; }
8080

8181
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.Configuration)]

src/Synapse/Synapse/Commands/ManagementCommands/SparkPool/NewAzureSynapseSparkPool.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,14 @@ public class NewAzureSynapseSparkPool : SynapseManagementCmdletBase
7878
[ValidateRange(3, 200)]
7979
public int NodeCount { get; set; }
8080

81+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
82+
HelpMessage = HelpMessages.IsolatedCompute)]
83+
public SwitchParameter EnableIsolatedCompute { get; set; }
84+
8185
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true,
8286
HelpMessage = HelpMessages.NodeSize)]
83-
[ValidateSet(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, IgnoreCase = true)]
84-
[PSArgumentCompleter(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large)]
87+
[ValidateSet(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, Management.Synapse.Models.NodeSize.XLarge, Management.Synapse.Models.NodeSize.XXLarge, Management.Synapse.Models.NodeSize.XXXLarge, IgnoreCase = true)]
88+
[PSArgumentCompleter(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, Management.Synapse.Models.NodeSize.XLarge, Management.Synapse.Models.NodeSize.XXLarge, Management.Synapse.Models.NodeSize.XXXLarge)]
8589
public string NodeSize { get; set; }
8690

8791
private SwitchParameter enableAutoScale;
@@ -219,6 +223,7 @@ public override void ExecuteCmdlet()
219223
{
220224
Location = existingWorkspace.Location,
221225
Tags = TagsConversionHelper.CreateTagDictionary(this.Tag, validate: true),
226+
IsComputeIsolationEnabled = EnableIsolatedCompute.IsPresent,
222227
NodeCount = this.enableAutoScale ? (int?)null : this.NodeCount,
223228
NodeSizeFamily = NodeSizeFamily.MemoryOptimized,
224229
NodeSize = NodeSize,

src/Synapse/Synapse/Commands/ManagementCommands/SparkPool/UpdateAzureSynapseSparkPool.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,14 @@ public class UpdateAzureSynapseSparkPool : SynapseManagementCmdletBase
110110
[ValidateRange(3, 200)]
111111
public int NodeCount { get; set; }
112112

113+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
114+
HelpMessage = HelpMessages.IsolatedCompute)]
115+
public bool? EnableIsolatedCompute { get; set; }
116+
113117
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
114118
HelpMessage = HelpMessages.NodeSize)]
115-
[ValidateSet(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, IgnoreCase = true)]
116-
[PSArgumentCompleter(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large)]
119+
[ValidateSet(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, Management.Synapse.Models.NodeSize.XLarge, Management.Synapse.Models.NodeSize.XXLarge, Management.Synapse.Models.NodeSize.XXXLarge, IgnoreCase = true)]
120+
[PSArgumentCompleter(Management.Synapse.Models.NodeSize.Small, Management.Synapse.Models.NodeSize.Medium, Management.Synapse.Models.NodeSize.Large, Management.Synapse.Models.NodeSize.XLarge, Management.Synapse.Models.NodeSize.XXLarge, Management.Synapse.Models.NodeSize.XXXLarge)]
117121
public string NodeSize { get; set; }
118122

119123
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
@@ -215,6 +219,7 @@ public override void ExecuteCmdlet()
215219
existingSparkPool.Tags = this.IsParameterBound(c => c.Tag) ? TagsConversionHelper.CreateTagDictionary(this.Tag, validate: true) : existingSparkPool.Tags;
216220
existingSparkPool.NodeCount = this.IsParameterBound(c => c.NodeCount) ? this.NodeCount : existingSparkPool.NodeCount;
217221
existingSparkPool.NodeSizeFamily = NodeSizeFamily.MemoryOptimized;
222+
existingSparkPool.IsComputeIsolationEnabled = this.EnableIsolatedCompute != null ? this.EnableIsolatedCompute : existingSparkPool.IsComputeIsolationEnabled ?? false;
218223
existingSparkPool.NodeSize = this.IsParameterBound(c => c.NodeSize) ? this.NodeSize : existingSparkPool.NodeSize;
219224
existingSparkPool.LibraryRequirements = this.IsParameterBound(c => c.LibraryRequirementsFilePath) ? CreateLibraryRequirements() : existingSparkPool.LibraryRequirements;
220225
existingSparkPool.SparkConfigProperties = this.IsParameterBound(c => c.SparkConfigFilePath) ? CreateSparkConfigProperties() : existingSparkPool.SparkConfigProperties;

src/Synapse/Synapse/Common/HelpMessages.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ public static class HelpMessages
8888

8989
public const string NodeCount = "Number of nodes to be allocated in the specified Spark pool.";
9090

91+
public const string IsolatedCompute = "The Isolate Compute option is only available with the XXXLarge (80 vCPU / 504 GB) node size. Enabling this option offers isolation for Apache Spark compute for untrusted services. Isolated compute costs the same as the non-isolated VM of the same size. If you expect to enable Isolated Compute for spark pool, ensure that your Synapse workspace is created in an isolated compute supported region, please refer to this document for more details: https://learn.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-pool-configurations#isolated-compute";
92+
9193
public const string NodeSize = "Number of core and memory to be used for nodes allocated in the specified Spark pool. This parameter must be specified when Auto-scale is disabled";
9294

9395
public const string EnableAutoScale = "Indicates whether Auto-scale should be enabled";

src/Synapse/Synapse/Models/DataPlaneModels/Artifact/SynapseAnalyticsArtifactsClient.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -507,12 +507,12 @@ public void UpdateLandingZoneCredential(string linkConnectionName, UpdateLanding
507507

508508
public LinkConnectionResource GetLinkConnection(string linkConnectionName)
509509
{
510-
return _linkConnectionClient.GetLinkConnection(linkConnectionName).Value;
510+
return _linkConnectionClient.Get(linkConnectionName).Value;
511511
}
512512

513513
public Pageable<LinkConnectionResource> GetLinkConnectionByWorkspace()
514514
{
515-
return _linkConnectionClient.ListLinkConnectionsByWorkspace();
515+
return _linkConnectionClient.ListByWorkspace();
516516
}
517517

518518
public void StartLinkConnection(string linkConnectionName)
@@ -527,13 +527,13 @@ public void StopLinkConnection(string linkConnectionName)
527527

528528
public void DeleteLinkConnection(string linkConnectionName)
529529
{
530-
_linkConnectionClient.DeleteLinkConnection(linkConnectionName);
530+
_linkConnectionClient.Delete(linkConnectionName);
531531
}
532532

533533
public LinkConnectionResource CreateOrUpdateLinkConnection(string linkConnectionName, string rawJsonContent)
534534
{
535535
LinkConnectionResource linkConnection = JsonConvert.DeserializeObject<LinkConnectionResource>(rawJsonContent);
536-
var response = _linkConnectionClient.CreateOrUpdateLinkConnection(linkConnectionName, linkConnection);
536+
var response = _linkConnectionClient.CreateOrUpdate(linkConnectionName, linkConnection);
537537
return response.Value;
538538
}
539539

src/Synapse/Synapse/Models/ManagementModels/SparkPool/PSSynapseSparkPool.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public PSSynapseSparkPool(BigDataPoolResourceInfo sparkPool)
118118
/// <summary>
119119
/// Gets the level of compute power that each node in the Big
120120
/// Data pool has. Possible values include: 'None', 'Small', 'Medium',
121-
/// 'Large'
121+
/// 'Large', 'XLarge', 'XXLarge', 'XXXLarge'
122122
/// </summary>
123123
public string NodeSize { get; set; }
124124

src/Synapse/Synapse/Models/SynapseConstants.cs

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,33 @@ public static class SynapseConstants
249249
Cores = 16,
250250
Memory = 112
251251
}
252+
},
253+
{
254+
NodeSize.XLarge,
255+
new ComputeNodeSize
256+
{
257+
Name = NodeSize.XLarge,
258+
Cores = 32,
259+
Memory = 224
260+
}
261+
},
262+
{
263+
NodeSize.XXLarge,
264+
new ComputeNodeSize
265+
{
266+
Name = NodeSize.XXLarge,
267+
Cores = 64,
268+
Memory = 368
269+
}
270+
},
271+
{
272+
NodeSize.XXXLarge,
273+
new ComputeNodeSize
274+
{
275+
Name = NodeSize.XXXLarge,
276+
Cores = 80,
277+
Memory = 424
278+
}
252279
}
253280
};
254281

0 commit comments

Comments
 (0)