[Databricks] DBFS Double Encryption Support

src/Databricks/Az.Databricks.format.ps1xml

@@ -8,6 +8,9 @@
       </ViewSelectedBy>
       <TableControl>
         <TableHeaders>
+          <TableColumnHeader>
+            <Label>PeeringName</Label>
+          </TableColumnHeader>
           <TableColumnHeader>
             <Label>ResourceGroupName</Label>
           </TableColumnHeader>
@@ -21,6 +24,9 @@
         <TableRowEntries>
           <TableRowEntry>
             <TableColumnItems>
+              <TableColumnItem>
+                <PropertyName>PeeringName</PropertyName>
+              </TableColumnItem>
               <TableColumnItem>
                 <PropertyName>ResourceGroupName</PropertyName>
               </TableColumnItem>
@@ -35,6 +41,28 @@
         </TableRowEntries>
       </TableControl>
     </View>
+    <View>
+      <Name>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.AddressSpace</Name>
+      <ViewSelectedBy>
+        <TypeName>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.AddressSpace</TypeName>
+      </ViewSelectedBy>
+      <TableControl>
+        <TableHeaders>
+          <TableColumnHeader>
+            <Label>AddressPrefix</Label>
+          </TableColumnHeader>
+        </TableHeaders>
+        <TableRowEntries>
+          <TableRowEntry>
+            <TableColumnItems>
+              <TableColumnItem>
+                <PropertyName>AddressPrefix</PropertyName>
+              </TableColumnItem>
+            </TableColumnItems>
+          </TableRowEntry>
+        </TableRowEntries>
+      </TableControl>
+    </View>
     <View>
       <Name>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.CreatedBy</Name>
       <ViewSelectedBy>
@@ -401,6 +429,108 @@
         </TableRowEntries>
       </TableControl>
     </View>
+    <View>
+      <Name>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.VirtualNetworkPeering</Name>
+      <ViewSelectedBy>
+        <TypeName>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.VirtualNetworkPeering</TypeName>
+      </ViewSelectedBy>
+      <TableControl>
+        <TableHeaders>
+          <TableColumnHeader>
+            <Label>Name</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>Type</Label>
+          </TableColumnHeader>
+        </TableHeaders>
+        <TableRowEntries>
+          <TableRowEntry>
+            <TableColumnItems>
+              <TableColumnItem>
+                <PropertyName>Name</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>Type</PropertyName>
+              </TableColumnItem>
+            </TableColumnItems>
+          </TableRowEntry>
+        </TableRowEntries>
+      </TableControl>
+    </View>
+    <View>
+      <Name>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.VirtualNetworkPeeringList</Name>
+      <ViewSelectedBy>
+        <TypeName>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.VirtualNetworkPeeringList</TypeName>
+      </ViewSelectedBy>
+      <TableControl>
+        <TableHeaders>
+          <TableColumnHeader>
+            <Label>NextLink</Label>
+          </TableColumnHeader>
+        </TableHeaders>
+        <TableRowEntries>
+          <TableRowEntry>
+            <TableColumnItems>
+              <TableColumnItem>
+                <PropertyName>NextLink</PropertyName>
+              </TableColumnItem>
+            </TableColumnItems>
+          </TableRowEntry>
+        </TableRowEntries>
+      </TableControl>
+    </View>
+    <View>
+      <Name>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.VirtualNetworkPeeringPropertiesFormat</Name>
+      <ViewSelectedBy>
+        <TypeName>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.VirtualNetworkPeeringPropertiesFormat</TypeName>
+      </ViewSelectedBy>
+      <TableControl>
+        <TableHeaders>
+          <TableColumnHeader>
+            <Label>AllowForwardedTraffic</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>AllowGatewayTransit</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>AllowVirtualNetworkAccess</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>PeeringState</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>ProvisioningState</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>UseRemoteGateway</Label>
+          </TableColumnHeader>
+        </TableHeaders>
+        <TableRowEntries>
+          <TableRowEntry>
+            <TableColumnItems>
+              <TableColumnItem>
+                <PropertyName>AllowForwardedTraffic</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>AllowGatewayTransit</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>AllowVirtualNetworkAccess</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>PeeringState</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>ProvisioningState</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>UseRemoteGateway</PropertyName>
+              </TableColumnItem>
+            </TableColumnItems>
+          </TableRowEntry>
+        </TableRowEntries>
+      </TableControl>
+    </View>
     <View>
       <Name>Microsoft.Azure.PowerShell.Cmdlets.Databricks.Models.Api20180401.Workspace</Name>
       <ViewSelectedBy>

src/Databricks/Az.Databricks.psd1

@@ -3,7 +3,7 @@
 #
 # Generated by: Microsoft Corporation
 #
-# Generated on: 6/17/2020
+# Generated on: 2020/7/29
 #
 
 @{
@@ -45,7 +45,7 @@ PowerShellVersion = '5.1'
 DotNetFrameworkVersion = '4.7.2'
 
 # Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only.
-# CLRVersion = ''
+# ClrVersion = ''
 
 # Processor architecture (None, X86, Amd64) required by this module
 # ProcessorArchitecture = ''
@@ -108,7 +108,7 @@ PrivateData = @{
         # IconUri = ''
 
         # ReleaseNotes of this module
-        ReleaseNotes = '* Added support for customer-managed keys.'
+        # ReleaseNotes = ''
 
         # Prerelease string of this module
         # Prerelease = ''

src/Databricks/Az.Databricks.psm1

@@ -24,17 +24,17 @@
       }
     }
     if(-not $accountsModule) {
-      $hasAdequateVersion = (Get-Module -Name $accountsName -ListAvailable | Where-Object { $_.Version -ge [System.Version]'1.7.4' } | Measure-Object).Count -gt 0
+      $hasAdequateVersion = (Get-Module -Name $accountsName -ListAvailable | Where-Object { $_.Version -ge [System.Version]'1.8.1' } | Measure-Object).Count -gt 0
       if($hasAdequateVersion) {
-        $accountsModule = Import-Module -Name $accountsName -MinimumVersion 1.7.4 -Scope Global -PassThru
+        $accountsModule = Import-Module -Name $accountsName -MinimumVersion 1.8.1 -Scope Global -PassThru
       }
     }
   }
 
   if(-not $accountsModule) {
-    Write-Error "`nThis module requires $accountsName version 1.7.4 or greater. For installation instructions, please see: https://docs.microsoft.com/en-us/powershell/azure/install-az-ps" -ErrorAction Stop
-  } elseif (($accountsModule.Version -lt [System.Version]'1.7.4') -and (-not $localAccounts)) {
-    Write-Error "`nThis module requires $accountsName version 1.7.4 or greater. An earlier version of Az.Accounts is imported in the current PowerShell session. If you are running test, please try to remove '.PSSharedModules' in your home directory. Otherwise please open a new PowerShell session and import this module again.`nAdditionally, this error could indicate that multiple incompatible versions of Azure PowerShell modules are installed on your system. For troubleshooting information, please see: https://aka.ms/azps-version-error" -ErrorAction Stop
+    Write-Error "`nThis module requires $accountsName version 1.8.1 or greater. For installation instructions, please see: https://docs.microsoft.com/en-us/powershell/azure/install-az-ps" -ErrorAction Stop
+  } elseif (($accountsModule.Version -lt [System.Version]'1.8.1') -and (-not $localAccounts)) {
+    Write-Error "`nThis module requires $accountsName version 1.8.1 or greater. An earlier version of Az.Accounts is imported in the current PowerShell session. If you are running test, please try to remove '.PSSharedModules' in your home directory. Otherwise please open a new PowerShell session and import this module again.`nAdditionally, this error could indicate that multiple incompatible versions of Azure PowerShell modules are installed on your system. For troubleshooting information, please see: https://aka.ms/azps-version-error" -ErrorAction Stop
   }
   Write-Information "Loaded Module '$($accountsModule.Name)'"
 

src/Databricks/ChangeLog.md

@@ -18,9 +18,10 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Added support for DBFS Double Encryption Support
 
 ## Version 0.1.1
-* Added support for customer-managed keys.
+* Added support for customer-managed keys
 
 ## Version 0.1.0
 * the first preview release

src/Databricks/check-dependencies.ps1

@@ -46,7 +46,7 @@ if(Test-Path -Path $localModulesPath) {
   $env:PSModulePath = "$localModulesPath$([IO.Path]::PathSeparator)$env:PSModulePath"
 }
 
-DownloadModule -predicate ($all -or $Accounts) -path $localModulesPath -moduleName 'Az.Accounts' -versionMinimum '1.7.4'
+DownloadModule -predicate ($all -or $Accounts) -path $localModulesPath -moduleName 'Az.Accounts' -versionMinimum '1.8.1'
 DownloadModule -predicate ($all -or $Pester) -path $localModulesPath -moduleName 'Pester' -requiredVersion '4.10.1'
 
 $tools = Join-Path $PSScriptRoot 'tools'

src/Databricks/custom/New-AzDatabricksWorkspace.ps1

@@ -71,13 +71,13 @@ function New-AzDatabricksWorkspace {
         [Parameter()]
         [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
         [System.String]
-        # The value which should be used for this field.
+        # The name of the Private Subnet within the Virtual Network.
         ${PrivateSubnetName},
 
         [Parameter()]
         [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
         [System.String]
-        # The value which should be used for this field.
+        # The name of a Public Subnet within the Virtual Network.
         ${PublicSubnetName},
 
         [Parameter()]
@@ -96,15 +96,21 @@ function New-AzDatabricksWorkspace {
         [Parameter()]
         [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
         [System.String]
-        # The value which should be used for this field.
+        # The ID of a Virtual Network where this Databricks Cluster should be created.
         ${VirtualNetworkId},
 
         [Parameter(HelpMessage = "Prepare the workspace for encryption. Enables the Managed Identity for managed storage account.")]
         [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
         [System.Management.Automation.SwitchParameter]
-        # The value which should be used for this field.
+        # Prepare the workspace for encryption. Enables the Managed Identity for managed storage account.
         ${PrepareEncryption},
 
+        [Parameter(ParameterSetName = 'CreateExpanded')]
+        [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
+        [System.Management.Automation.SwitchParameter]
+        # A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest.
+        ${RequireInfrastructureEncryption},
+
         [Parameter()]
         [Alias('AzureRMContext', 'AzureCredential')]
         [ValidateNotNull()]

src/Databricks/custom/Update-AzDatabricksWorkspace.ps1

@@ -54,7 +54,7 @@ function Update-AzDatabricksWorkspace {
         [Parameter(HelpMessage = "Prepare the workspace for encryption. Enables the Managed Identity for managed storage account.")]
         [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
         [System.Management.Automation.SwitchParameter]
-        # The value which should be used for this field.
+        # Prepare the workspace for encryption. Enables the Managed Identity for managed storage account.
         ${PrepareEncryption},
 
         [Parameter(HelpMessage = "The encryption keySource (provider). Possible values (case-insensitive): Default, Microsoft.Keyvault")]

src/Databricks/exports/Get-AzDatabricksWorkspace.ps1

@@ -50,6 +50,7 @@ To create the parameters described below, construct a hash table containing the
 
 INPUTOBJECT <IDatabricksIdentity>: Identity Parameter
   [Id <String>]: Resource identity path
+  [PeeringName <String>]: The name of the workspace vNet peering.
   [ResourceGroupName <String>]: The name of the resource group. The name is case insensitive.
   [SubscriptionId <String>]: The ID of the target subscription.
   [WorkspaceName <String>]: The name of the workspace.

src/Databricks/exports/New-AzDatabricksWorkspace.ps1

@@ -88,13 +88,13 @@ param(
     [Parameter()]
     [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
     [System.String]
-    # The value which should be used for this field.
+    # The name of the Private Subnet within the Virtual Network.
     ${PrivateSubnetName},
 
     [Parameter()]
     [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
     [System.String]
-    # The value which should be used for this field.
+    # The name of a Public Subnet within the Virtual Network.
     ${PublicSubnetName},
 
     [Parameter()]
@@ -113,7 +113,7 @@ param(
     [Parameter()]
     [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
     [System.String]
-    # The value which should be used for this field.
+    # The ID of a Virtual Network where this Databricks Cluster should be created.
     ${VirtualNetworkId},
 
     [Parameter()]
@@ -123,6 +123,12 @@ param(
     # Enables the Managed Identity for managed storage account.
     ${PrepareEncryption},
 
+    [Parameter()]
+    [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
+    [System.Management.Automation.SwitchParameter]
+    # A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest.
+    ${RequireInfrastructureEncryption},
+
     [Parameter()]
     [Alias('AzureRMContext', 'AzureCredential')]
     [ValidateNotNull()]

src/Databricks/exports/ProxyCmdletDefinitions.ps1

@@ -50,6 +50,7 @@ To create the parameters described below, construct a hash table containing the
 
 INPUTOBJECT <IDatabricksIdentity>: Identity Parameter
   [Id <String>]: Resource identity path
+  [PeeringName <String>]: The name of the workspace vNet peering.
   [ResourceGroupName <String>]: The name of the resource group. The name is case insensitive.
   [SubscriptionId <String>]: The ID of the target subscription.
   [WorkspaceName <String>]: The name of the workspace.
@@ -217,6 +218,7 @@ To create the parameters described below, construct a hash table containing the
 
 INPUTOBJECT <IDatabricksIdentity>: Identity Parameter
   [Id <String>]: Resource identity path
+  [PeeringName <String>]: The name of the workspace vNet peering.
   [ResourceGroupName <String>]: The name of the resource group. The name is case insensitive.
   [SubscriptionId <String>]: The ID of the target subscription.
   [WorkspaceName <String>]: The name of the workspace.
@@ -450,13 +452,13 @@ param(
     [Parameter()]
     [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
     [System.String]
-    # The value which should be used for this field.
+    # The name of the Private Subnet within the Virtual Network.
     ${PrivateSubnetName},
 
     [Parameter()]
     [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
     [System.String]
-    # The value which should be used for this field.
+    # The name of a Public Subnet within the Virtual Network.
     ${PublicSubnetName},
 
     [Parameter()]
@@ -475,7 +477,7 @@ param(
     [Parameter()]
     [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
     [System.String]
-    # The value which should be used for this field.
+    # The ID of a Virtual Network where this Databricks Cluster should be created.
     ${VirtualNetworkId},
 
     [Parameter()]
@@ -485,6 +487,12 @@ param(
     # Enables the Managed Identity for managed storage account.
     ${PrepareEncryption},
 
+    [Parameter()]
+    [Microsoft.Azure.PowerShell.Cmdlets.Databricks.Category('Body')]
+    [System.Management.Automation.SwitchParameter]
+    # A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest.
+    ${RequireInfrastructureEncryption},
+
     [Parameter()]
     [Alias('AzureRMContext', 'AzureCredential')]
     [ValidateNotNull()]
@@ -631,6 +639,7 @@ To create the parameters described below, construct a hash table containing the
 
 INPUTOBJECT <IDatabricksIdentity>: Identity parameter.
   [Id <String>]: Resource identity path
+  [PeeringName <String>]: The name of the workspace vNet peering.
   [ResourceGroupName <String>]: The name of the resource group. The name is case insensitive.
   [SubscriptionId <String>]: The ID of the target subscription.
   [WorkspaceName <String>]: The name of the workspace.

src/Databricks/exports/Remove-AzDatabricksWorkspace.ps1

@@ -35,6 +35,7 @@ To create the parameters described below, construct a hash table containing the
 
 INPUTOBJECT <IDatabricksIdentity>: Identity Parameter
   [Id <String>]: Resource identity path
+  [PeeringName <String>]: The name of the workspace vNet peering.
   [ResourceGroupName <String>]: The name of the resource group. The name is case insensitive.
   [SubscriptionId <String>]: The ID of the target subscription.
   [WorkspaceName <String>]: The name of the workspace.