[Synapse] Make Synapse PowerShell consume track 2 Spark SDK (#12202)

* update NewAzureSynapseSparkPool

* fix NewAzureSynapseSparkPool

* fix NewAzureSynapseSparkPool for auto scale

* fix New-AzSynapseSparkPool for auto scale

* add New-AzSynapseSparkPool help doc

* fix New-AzSynapseSparkPool help doc

* update parameter set name

* track2 v1.0

* track2 v2.0

* track2 v2.1

* Make Synapse PowerShell consume track 2 SDK

* Make Synapse PowerShell consume track 2 SDK

* add change log

* fix proj

* Make Synapse PowerShell consume track 2 SDK

* fix AzureSessionCredential

* update changelog

* fix AzureSessionCredential

* Add required assembly Azure.Analytics.Synapse.Spark.dll to psd1

* change tab to space

Co-authored-by: Wan Yang <[email protected]>
Co-authored-by: erich-wang <[email protected]>

Author: 3 people

src/Synapse/Synapse/Az.Synapse.psd1

@@ -57,7 +57,7 @@ RequiredModules = @(@{ModuleName = 'Az.Accounts'; ModuleVersion = '1.9.0'; })
 
 # Assemblies that must be loaded prior to importing this module
 RequiredAssemblies = 'Microsoft.Azure.Management.Synapse.dll', 
-               'Microsoft.Azure.Synapse.dll'
+               'Azure.Analytics.Synapse.Spark.dll'
 
 # Script files (.ps1) that are run in the caller's environment prior to importing this module.
 # ScriptsToProcess = @()

src/Synapse/Synapse/ChangeLog.md

@@ -19,6 +19,17 @@
 -->
 ## Upcoming Release
 
+* Changed some property names and types of output for the following cmdlets
+    - For `Get-AzSynapseSparkJob`, `Submit-AzSynapseSparkJob`, ` Get-AzSynapseSparkSession` and `Start-AzSynapseSparkSession` cmdlet
+        - Change JobType's type from `string` to `SparkJobType?`
+        - Change AppInfo's type from `IDictionary<string, string>` to `IReadOnlyDictionary<string, string>`
+        - Change ErrorInfo's type from `IList<ErrorInformation>` to `IReadOnlyList<SparkServiceError>`
+        - Change Log's type from `IList<string>` to `IReadOnlyList<string>`
+        - Change `Scheduler` to `Scheduler`
+        - Change `PluginInfo` to `Plugin`
+        - Change `ErrorInfo` to `Errors`
+        - Change `Log` to `LogLines`
+
 ## Version 0.1.1
 
 * Added support for operation of Synapse FirewallRule

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkJob/GetAzureSynapseSparkJob.cs

@@ -1,4 +1,5 @@
-using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Azure.Analytics.Synapse.Spark;
+using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
 using Microsoft.Azure.Commands.Synapse.Common;
 using Microsoft.Azure.Commands.Synapse.Models;
 using Microsoft.Azure.Management.Internal.Resources.Utilities.Models;
@@ -12,7 +13,7 @@ namespace Microsoft.Azure.Commands.Synapse
     [Cmdlet(VerbsCommon.Get, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkJob,
         DefaultParameterSetName = GetSparkJobsByIdParameterSetName)]
     [OutputType(typeof(PSSynapseSparkJob))]
-    public class GetAzureSynapseSparkJob : SynapseCmdletBase
+    public class GetAzureSynapseSparkJob : SynapseSparkCmdletBase
     {
         private const string GetSparkJobsByIdParameterSetName = "GetSparkJobsByIdParameterSet";
         private const string GetSparkJobsByIdFromParentObjectParameterSetName = "GetSparkJobsByIdFromParentObjectParameterSet";
@@ -21,7 +22,7 @@ public class GetAzureSynapseSparkJob : SynapseCmdletBase
             Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
         [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
         [ValidateNotNullOrEmpty]
-        public string WorkspaceName { get; set; }
+        public override string WorkspaceName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = GetSparkJobsByIdParameterSetName,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
@@ -30,7 +31,7 @@ public class GetAzureSynapseSparkJob : SynapseCmdletBase
             "ResourceGroupName",
             nameof(WorkspaceName))]
         [ValidateNotNullOrEmpty]
-        public string SparkPoolName { get; set; }
+        public override string SparkPoolName { get; set; }
 
         [Parameter(ValueFromPipeline = true, ParameterSetName = GetSparkJobsByIdFromParentObjectParameterSetName,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolObject)]
@@ -72,12 +73,12 @@ public override void ExecuteCmdlet()
             if (this.IsParameterBound(c => c.LivyId))
             {
                 // Get for single Spark batch job
-                WriteObject(new PSSynapseSparkJob(SynapseAnalyticsClient.GetSparkBatchJob(this.WorkspaceName, this.SparkPoolName, this.LivyId)));
+                WriteObject(new PSSynapseSparkJob(SynapseAnalyticsClient.GetSparkBatchJob(this.LivyId)));
             }
             else
             {
                 // List all Spark batch jobs in given Spark pool
-                var batchJobs = SynapseAnalyticsClient.ListSparkBatchJobs(this.WorkspaceName, this.SparkPoolName)
+                var batchJobs = SynapseAnalyticsClient.ListSparkBatchJobs()
                     .Select(element => new PSSynapseSparkJob(element));
                 if (!string.IsNullOrEmpty(this.Name))
                 {

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkJob/StopAzureSynapseSparkJob.cs

@@ -10,7 +10,7 @@ namespace Microsoft.Azure.Commands.Synapse
 {
     [Cmdlet(VerbsLifecycle.Stop, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkJob, DefaultParameterSetName = StopSparkJobByIdParameterSetName, SupportsShouldProcess = true)]
     [OutputType(typeof(bool))]
-    public class StopAzureSynapseSparkJob : SynapseCmdletBase
+    public class StopAzureSynapseSparkJob : SynapseSparkCmdletBase
     {
         private const string StopSparkJobByIdParameterSetName = "StopSparkJobByIdParameterSet";
         private const string StopSparkJobByIdFromParentObjectParameterSet = "StopSparkJobByIdFromParentObjectParameterSet";
@@ -20,7 +20,7 @@ public class StopAzureSynapseSparkJob : SynapseCmdletBase
             Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
         [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
         [ValidateNotNullOrEmpty]
-        public string WorkspaceName { get; set; }
+        public override string WorkspaceName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = StopSparkJobByIdParameterSetName,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
@@ -29,7 +29,7 @@ public class StopAzureSynapseSparkJob : SynapseCmdletBase
             "ResourceGroupName",
             nameof(WorkspaceName))]
         [ValidateNotNullOrEmpty]
-        public string SparkPoolName { get; set; }
+        public override string SparkPoolName { get; set; }
 
         [Parameter(ValueFromPipeline = true, ParameterSetName = StopSparkJobByIdFromParentObjectParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolObject)]
@@ -81,7 +81,7 @@ public override void ExecuteCmdlet()
                 LivyId.ToString(),
                 () =>
                 {
-                    SynapseAnalyticsClient.CancelSparkBatchJob(WorkspaceName, SparkPoolName, LivyId, waitForCompletion: false);
+                    SynapseAnalyticsClient.CancelSparkBatchJob(LivyId, waitForCompletion: false);
                     if (PassThru)
                     {
                         WriteObject(true);

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkJob/SubmitAzureSynapseSparkJob.cs

@@ -1,11 +1,11 @@
-using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Azure.Analytics.Synapse.Spark.Models;
+using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
 using Microsoft.Azure.Commands.Synapse.Common;
 using Microsoft.Azure.Commands.Synapse.Models;
 using Microsoft.Azure.Commands.Synapse.Models.Exceptions;
 using Microsoft.Azure.Commands.Synapse.Properties;
 using Microsoft.Azure.Management.Internal.Resources.Utilities.Models;
 using Microsoft.Azure.Management.Synapse.Models;
-using Microsoft.Azure.Synapse.Models;
 using Microsoft.WindowsAzure.Commands.Utilities.Common;
 using System.Collections;
 using System.Collections.Generic;
@@ -16,7 +16,7 @@ namespace Microsoft.Azure.Commands.Synapse
 {
     [Cmdlet(VerbsLifecycle.Submit, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkJob, DefaultParameterSetName = RunSparkJobParameterSetName)]
     [OutputType(typeof(PSSynapseSparkJob))]
-    public class SubmitAzureSynapseSparkJob : SynapseCmdletBase
+    public class SubmitAzureSynapseSparkJob : SynapseSparkCmdletBase
     {
         private const string RunSparkJobParameterSetName = nameof(RunSparkJobParameterSetName);
         private const string RunSparkJobByParentObjectParameterSet = nameof(RunSparkJobByParentObjectParameterSet);
@@ -25,7 +25,7 @@ public class SubmitAzureSynapseSparkJob : SynapseCmdletBase
             Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
         [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
         [ValidateNotNullOrEmpty]
-        public string WorkspaceName { get; set; }
+        public override string WorkspaceName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RunSparkJobParameterSetName,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
@@ -34,7 +34,7 @@ public class SubmitAzureSynapseSparkJob : SynapseCmdletBase
             "ResourceGroupName",
             nameof(WorkspaceName))]
         [ValidateNotNullOrEmpty]
-        public string SparkPoolName { get; set; }
+        public override string SparkPoolName { get; set; }
 
         [Parameter(ValueFromPipeline = true, ParameterSetName = RunSparkJobByParentObjectParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolObject)]
@@ -147,16 +147,12 @@ public override void ExecuteCmdlet()
 
             Utils.CategorizedFiles(this.ReferenceFile, out IList<string> jars, out IList<string> files);
             bool isSparkDotNet = this.Language == LanguageType.SparkDotNet;
-            var batchRequest = new ExtendedLivyBatchRequest
+            var batchRequest = new SparkBatchJobOptions(this.Name, isSparkDotNet ? SynapseConstants.SparkDotNetJarFile : this.MainDefinitionFile)
             {
-                Name = this.Name,
-                File = isSparkDotNet
-                    ? SynapseConstants.SparkDotNetJarFile
-                    : this.MainDefinitionFile,
                 ClassName = isSparkDotNet
                     ? SynapseConstants.SparkDotNetClassName
                     : (this.Language == LanguageType.PySpark ? null : this.MainClassName),
-                Args = isSparkDotNet
+                Arguments = isSparkDotNet
                     ? new List<string> { this.MainDefinitionFile, this.MainClassName }
                         .Concat(this.CommandLineArgument ?? new string[0]).ToArray()
                     : this.CommandLineArgument,
@@ -165,30 +161,30 @@ public override void ExecuteCmdlet()
                 Archives = isSparkDotNet
                     ? new List<string> { $"{this.MainDefinitionFile}#{SynapseConstants.SparkDotNetUdfsFolderName}" }
                     : null,
-                Conf = this.Configuration?.ToDictionary(),
+                Configuration = this.Configuration?.ToDictionary(),
                 ExecutorMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g",
                 ExecutorCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores,
                 DriverMemory = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Memory + "g",
                 DriverCores = SynapseConstants.ComputeNodeSizes[this.ExecutorSize].Cores,
-                NumExecutors = this.ExecutorCount
+                ExecutorCount = this.ExecutorCount
             };
 
             // Ensure the relative path of UDFs is add to "--conf".
             if (isSparkDotNet)
             {
-                batchRequest.Conf = batchRequest.Conf ?? new Dictionary<string, string>();
+                batchRequest.Configuration = batchRequest.Configuration ?? new Dictionary<string, string>();
                 string udfsRelativePath = "./" + SynapseConstants.SparkDotNetUdfsFolderName;
-                batchRequest.Conf.TryGetValue(SynapseConstants.SparkDotNetAssemblySearchPathsKey, out string pathValue);
+                batchRequest.Configuration.TryGetValue(SynapseConstants.SparkDotNetAssemblySearchPathsKey, out string pathValue);
                 var paths = pathValue?.Split(',').Select(path => path.Trim()).Where(path => !string.IsNullOrEmpty(path)).ToList() ?? new List<string>();
                 if (!paths.Contains(udfsRelativePath))
                 {
                     paths.Add(udfsRelativePath);
                 }
 
-                batchRequest.Conf[SynapseConstants.SparkDotNetAssemblySearchPathsKey] = string.Join(",", paths);
+                batchRequest.Configuration[SynapseConstants.SparkDotNetAssemblySearchPathsKey] = string.Join(",", paths);
             }
 
-            var jobInformation = SynapseAnalyticsClient.SubmitSparkBatchJob(this.WorkspaceName, this.SparkPoolName, batchRequest, waitForCompletion:false);
+            var jobInformation = SynapseAnalyticsClient.SubmitSparkBatchJob(batchRequest, waitForCompletion:false);
             WriteObject(new PSSynapseSparkJob(jobInformation));
         }
     }

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkJob/WaitAzureSynapseSparkJob.cs

@@ -12,7 +12,7 @@ namespace Microsoft.Azure.Commands.Synapse
 {
     [Cmdlet(VerbsLifecycle.Wait, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkJob, DefaultParameterSetName = WaitSparkJobByIdParameterSetName)]
     [OutputType(typeof(bool))]
-    public class WaitAzureSynapseSparkJob : SynapseCmdletBase
+    public class WaitAzureSynapseSparkJob : SynapseSparkCmdletBase
     {
         private const string WaitSparkJobByIdParameterSetName = "WaitSparkJobByIdParameterSet";
         private const string WaitSparkJobByIdFromParentObjectParameterSet = "WaitSparkJobByIdFromParentObjectParameterSet";
@@ -22,7 +22,7 @@ public class WaitAzureSynapseSparkJob : SynapseCmdletBase
             Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
         [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
         [ValidateNotNullOrEmpty]
-        public string WorkspaceName { get; set; }
+        public override string WorkspaceName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = WaitSparkJobByIdParameterSetName,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
@@ -31,7 +31,7 @@ public class WaitAzureSynapseSparkJob : SynapseCmdletBase
             "ResourceGroupName",
             nameof(WorkspaceName))]
         [ValidateNotNullOrEmpty]
-        public string SparkPoolName { get; set; }
+        public override string SparkPoolName { get; set; }
 
         [Parameter(ValueFromPipeline = true, ParameterSetName = WaitSparkJobByIdFromParentObjectParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolObject)]
@@ -80,12 +80,10 @@ public override void ExecuteCmdlet()
                 this.LivyId = this.IsParameterBound(c => c.LivyId) ? this.LivyId : this.SparkJobObject.Id.Value;
             }
 
-            var sparkJob = this.SynapseAnalyticsClient.GetSparkBatchJob(this.WorkspaceName, this.SparkPoolName, this.LivyId);
+            var sparkJob = this.SynapseAnalyticsClient.GetSparkBatchJob(this.LivyId);
             try
             {
                 sparkJob = this.SynapseAnalyticsClient.PollSparkBatchJobExecution(
-                    this.WorkspaceName,
-                    this.SparkPoolName,
                     sparkJob,
                     this.WaitIntervalInSeconds,
                     this.TimeoutInSeconds,

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkSession/GetAzureSynapseSparkSession.cs

@@ -11,7 +11,7 @@ namespace Microsoft.Azure.Commands.Synapse
 {
     [Cmdlet(VerbsCommon.Get, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkSession, DefaultParameterSetName = GetByNameParameterSet)]
     [OutputType(typeof(PSSynapseSparkSession))]
-    public class GetAzureSynapseSparkSession : SynapseCmdletBase
+    public class GetAzureSynapseSparkSession : SynapseSparkCmdletBase
     {
         private const string GetByNameParameterSet = "GetByNameParameterSet";
         private const string GetByParentObjectParameterSet = "GetByParentObjectParameterSet";
@@ -20,7 +20,7 @@ public class GetAzureSynapseSparkSession : SynapseCmdletBase
             Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
         [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
         [ValidateNotNullOrEmpty]
-        public string WorkspaceName { get; set; }
+        public override string WorkspaceName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = GetByNameParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
@@ -29,7 +29,7 @@ public class GetAzureSynapseSparkSession : SynapseCmdletBase
             "ResourceGroupName",
             nameof(WorkspaceName))]
         [ValidateNotNullOrEmpty]
-        public string SparkPoolName { get; set; }
+        public override string SparkPoolName { get; set; }
 
         [Parameter(ValueFromPipeline = true, ParameterSetName = GetByParentObjectParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolObject)]
@@ -70,12 +70,12 @@ public override void ExecuteCmdlet()
 
             if (this.IsParameterBound(c => c.LivyId))
             {
-                var result = new PSSynapseSparkSession(this.SynapseAnalyticsClient.GetSparkSession(this.WorkspaceName, this.SparkPoolName, this.LivyId));
+                var result = new PSSynapseSparkSession(this.SynapseAnalyticsClient.GetSparkSession(this.LivyId));
                 WriteObject(result);
             }
             else
             {
-                var result = this.SynapseAnalyticsClient.ListSparkSessions(this.WorkspaceName, this.SparkPoolName).Select(r => new PSSynapseSparkSession(r));
+                var result = this.SynapseAnalyticsClient.ListSparkSessions().Select(r => new PSSynapseSparkSession(r));
                 if (!string.IsNullOrEmpty(this.Name))
                 {
                     result = result.Where(r => this.Name.Equals(r.Name, StringComparison.OrdinalIgnoreCase));

src/Synapse/Synapse/Commands/DataPlaneCommands/SparkSession/ResetAzureSynapseSparkSessionTimeout.cs

@@ -10,7 +10,7 @@ namespace Microsoft.Azure.Commands.Synapse
 {
     [Cmdlet(VerbsCommon.Reset, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkSession + SynapseConstants.Timeout, DefaultParameterSetName = ResetByIdParameterSet, SupportsShouldProcess = true)]
     [OutputType(typeof(bool))]
-    public class ResetAzureSynapseSparkSessionTimeout : SynapseCmdletBase
+    public class ResetAzureSynapseSparkSessionTimeout : SynapseSparkCmdletBase
     {
         private const string ResetByIdParameterSet = "ResetByNameParameterSet";
         private const string ResetByParentObjectParameterSet = "ResetByParentObjectParameterSet";
@@ -20,7 +20,7 @@ public class ResetAzureSynapseSparkSessionTimeout : SynapseCmdletBase
             Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
         [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
         [ValidateNotNullOrEmpty]
-        public string WorkspaceName { get; set; }
+        public override string WorkspaceName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = ResetByIdParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SparkPoolName)]
@@ -29,7 +29,7 @@ public class ResetAzureSynapseSparkSessionTimeout : SynapseCmdletBase
             "ResourceGroupName",
             nameof(WorkspaceName))]
         [ValidateNotNullOrEmpty]
-        public string SparkPoolName { get; set; }
+        public override string SparkPoolName { get; set; }
 
         [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = ResetByIdParameterSet,
             Mandatory = true, HelpMessage = HelpMessages.SessionId)]
@@ -74,7 +74,7 @@ public override void ExecuteCmdlet()
 
             if (this.ShouldProcess(this.LivyId.ToString(), string.Format(Resources.ResettingSynapseSparkSessionTimeout, this.LivyId)))
             {
-                this.SynapseAnalyticsClient.ResetSparkSessionTimeout(this.WorkspaceName, this.SparkPoolName, this.LivyId);
+                this.SynapseAnalyticsClient.ResetSparkSessionTimeout(this.LivyId);
                 if (this.PassThru.IsPresent)
                 {
                     WriteObject(true);