Merge pull request #235 from SQLPlayer/develop

NowinskiK · web-flow · commit 01432506802a · 2022-10-24T13:08:14.000+01:00
The module accepts **Credentials** type of object (when loading from files), but the deployment is skipped and not supported yet. #156
diff --git a/README.md b/README.md
@@ -36,11 +36,10 @@ The main advantage of the module is the ability to publish all the Azure Data Fa
 * Build function to support validation of files, dependencies and config
 * Test connections (Linked Services)
 * Generates mermaid dependencies diagram to be used in MarkDown type of documents
-  
-The following features coming in the future:
-* Unit Tests of selected Pipelines and Linked Services
 
-> The module publishes code, which is created and maintained by ADF in a code repository, when configured.
+# Known issues
+
+The module accepts **Credentials** type of object (when loading from files), but the deployment is skipped and not supported yet. [Read more here](https://github.com/SQLPlayer/azure.datafactory.tools/issues/156).
 
 # Overview
 
diff --git a/adhoc/Test-AdfArmTemplate-debug.ps1 b/adhoc/Test-AdfArmTemplate-debug.ps1
@@ -2,7 +2,7 @@ Import-Module ".\azure.datafactory.tools.psd1" -Force
 $VerbosePreference = 'Continue'
 $ErrorActionPreference = 'Stop'
 
-$folder = 'd:\GitHub\mrpaulandrew\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\DataFactory'
+$folder = 'X:\!WORK\GitHub\!SQLPlayer\azure.datafactory.tools\test\adf1'
 Export-AdfToArmTemplate -RootFolder $folder
 
 Publish-AdfV2FromJson -RootFolder $folder -ResourceGroupName 'rg-devops' -DataFactoryName 'padf2022bits' -Location 'uksouth'
@@ -14,4 +14,6 @@ $r
 Get-AzContext
 
 $adf = Import-AdfFromFolder -FactoryName 'abc' -RootFolder $folder
-Get-AdfDocDiagram -adf $adf
+$doc = Get-AdfDocDiagram -adf $adf
+$doc | Set-Content 'diagram.md'
+$doc
diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1
@@ -3,7 +3,7 @@
 #
 # Generated by: Kamil Nowinski @ SQLPlayer.net
 #
-# Generated on: 25/04/2022
+# Generated on: 10/10/2022
 #
 
 @{
@@ -12,7 +12,7 @@
     RootModule = 'azure.datafactory.tools.psm1'
     
     # Version number of this module.
-    ModuleVersion = '0.97.0'
+    ModuleVersion = '0.99.0'
     
     # Supported PSEditions
     # CompatiblePSEditions = @()
diff --git a/changelog.md b/changelog.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.99.0] - 2022-10-24 
+### Fixed
+* The module accepts **Credentials** type of object (when loading from files), but the deployment is skipped and not supported yet. #156
+
 ## [0.97.0] - 2022-04-25
 ### Fixed
 * `Publish-AdfV2UsingArm` cmdlet to make it compatible with PS 5.1
diff --git a/diagram.md b/diagram.md
@@ -0,0 +1,17 @@
+# Option 1 (:::)
+::: mermaid
+graph LR
+dataset.movie_dataflow_sink --> linkedService.BlobSampleData
+dataset.movie_dataflow_source --> linkedService.BlobSampleData
+dataflow.MovieDemo --> dataset.movie_dataflow_sink
+dataflow.MovieDemo --> dataset.movie_dataflow_source
+:::
+
+# Option 2 (```)
+``` mermaid
+graph LR
+dataset.movie_dataflow_sink --> linkedService.BlobSampleData
+dataset.movie_dataflow_source --> linkedService.BlobSampleData
+dataflow.MovieDemo --> dataset.movie_dataflow_sink
+dataflow.MovieDemo --> dataset.movie_dataflow_source
+```
diff --git a/private/Adf.class.ps1 b/private/Adf.class.ps1
@@ -17,14 +17,15 @@ class Adf {
     [System.Collections.ArrayList] $Factories = @{}
     [System.Collections.ArrayList] $ManagedVirtualNetwork = @{}
     [System.Collections.ArrayList] $ManagedPrivateEndpoints = @{}
+    [System.Collections.ArrayList] $Credentials = @{}
     [string] $Location = ""
     [AdfGlobalProp] $GlobalFactory = [AdfGlobalProp]::new()
     [AdfPublishOption] $PublishOptions
     $ArmTemplateJson
     
     [System.Collections.ArrayList] AllObjects()
     {
-        return $this.LinkedServices + $this.Pipelines + $this.DataSets + $this.DataFlows + $this.Triggers + $this.IntegrationRuntimes + $this.Factories + $this.ManagedVirtualNetwork + $this.ManagedPrivateEndpoints
+        return $this.LinkedServices + $this.Pipelines + $this.DataSets + $this.DataFlows + $this.Triggers + $this.IntegrationRuntimes + $this.Factories + $this.ManagedVirtualNetwork + $this.ManagedPrivateEndpoints + $this.Credentials
     }
 
     [hashtable] GetObjectsByFullName([string]$pattern)
diff --git a/private/AdfObject.class.ps1 b/private/AdfObject.class.ps1
@@ -83,7 +83,7 @@ class AdfObject {
         return $ofn
     }
 
-    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory', 'managedVirtualNetwork', 'managedPrivateEndpoint')
+    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory', 'managedVirtualNetwork', 'managedPrivateEndpoint', 'credential')
 
     static AssertType ([string] $Type)
     {
diff --git a/private/ConvertTo-AdfType.ps1 b/private/ConvertTo-AdfType.ps1
@@ -15,6 +15,7 @@ function ConvertTo-AdfType {
         'Microsoft.DataFactory/factories/dataflows'              { $resType = 'dataflow' }
         'Microsoft.DataFactory/factories/linkedservices'         { $resType = 'linkedService' }
         'Microsoft.DataFactory/factories/triggers'               { $resType = 'trigger' }
+        'Microsoft.DataFactory/factories/credentials'            { $resType = 'credential' }
         'Microsoft.DataFactory/factories/managedVirtualNetworks' { $resType = 'managedVirtualNetwork' }
         'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints' { $resType = 'managedVirtualNetwork\default\managedPrivateEndpoint' }
         'Microsoft.DataFactory/factories'               { $resType = 'factory' }
diff --git a/private/Deploy-AdfObjectOnly.ps1 b/private/Deploy-AdfObjectOnly.ps1
@@ -21,6 +21,7 @@ function Deploy-AdfObjectOnly {
 
     $type = $obj.Type
     if ($script:PublishMethod -eq "AzResource") { $type = "AzResource" }
+    if ($obj.Type -eq "credential") { $type = $obj.Type }
     # Global parameters is being deployed with different method:
     if ($obj.Type -eq "factory") { $type = "GlobalParameters" }
 
@@ -121,6 +122,11 @@ function Deploy-AdfObjectOnly {
             -DefinitionFile $obj.FileName `
             -Force | Out-Null
         }
+        'credential'
+        {
+            Write-Warning "Credentials are not yet supported. The deployment for the object is skipped."
+            Write-Warning "Any reference to the object causes error, unless to deploy it before."
+        }
         'AzResource'
         {
             $resType = Get-AzureResourceType $obj.Type
diff --git a/private/Get-AdfObjectByName.ps1 b/private/Get-AdfObjectByName.ps1
@@ -35,6 +35,10 @@ function Get-AdfObjectByName {
         {
             $r = $adf.Triggers | Where-Object { $_.Name -eq $name } | Select-Object -First 1
         }
+        'Credential'
+        {
+            $r = $adf.Credentials | Where-Object { $_.Name -eq $name } | Select-Object -First 1
+        }
         'Factory'
         {
             $r = $adf.Factories | Where-Object { $_.Name -eq $name } | Select-Object -First 1
diff --git a/private/Get-AzureResourceType.ps1 b/private/Get-AzureResourceType.ps1
@@ -20,6 +20,7 @@ function Get-AzureResourceType {
         'managedVirtualNetwork' { $resType = 'Microsoft.DataFactory/factories/managedVirtualNetworks' }
         'managedVirtualNetwork\default\managedPrivateEndpoint' { $resType = 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints' }
         'factory'               { $resType = 'Microsoft.DataFactory/factories' }
+        'credential'            { $resType = 'Microsoft.DataFactory/credentials' }
         default                 { Write-Error "ADFT0016: Type '$Type' is not supported." }
     }
 
diff --git a/public/Import-AdfFromFolder.ps1 b/public/Import-AdfFromFolder.ps1
@@ -55,6 +55,8 @@ function Import-AdfFromFolder {
     Write-Host ("Managed VNet: {0} object(s) loaded." -f $adf.ManagedVirtualNetwork.Count)
     Import-AdfObjects -Adf $adf -All $adf.ManagedPrivateEndpoints -RootFolder $RootFolder -SubFolder "managedVirtualNetwork\default\managedPrivateEndpoint" | Out-Null
     Write-Host ("Managed Private Endpoints: {0} object(s) loaded." -f $adf.ManagedPrivateEndpoints.Count)
+    Import-AdfObjects -Adf $adf -All $adf.Credentials -RootFolder $RootFolder -SubFolder "credential" | Out-Null
+    Write-Host ("Credentials: {0} object(s) loaded." -f $adf.Credentials.Count)
     Import-AdfObjects -Adf $adf -All $adf.Factories -RootFolder $RootFolder -SubFolder "factory" | Out-Null
     Write-Host ("Factories: {0} object(s) loaded." -f $adf.Factories.Count)
 
diff --git a/test/Export-AdfToArmTemplate.Tests.ps1 b/test/Export-AdfToArmTemplate.Tests.ps1
@@ -24,7 +24,6 @@ InModuleScope azure.datafactory.tools {
     Write-Host $script:RootFolder
     
     $VerbosePreference = 'Continue'
-    $script:RootFolder = 'C:\Users\kamil\AppData\Local\Temp\ADFTools-3oirmp1y.vrn\BigFactorySample2'
 
     Describe 'Prerequisites of Export-AdfToArmTemplate' -Tag 'Unit' {
         It 'no files *.json!' {
diff --git a/test/Get-AdfFromService.Tests.ps1 b/test/Get-AdfFromService.Tests.ps1
@@ -11,11 +11,19 @@ InModuleScope azure.datafactory.tools {
     Import-Module -Name $testHelperPath -Force
 
     # Variables for use in tests
+    $script:adfName = 'SQLPlayerDemo-UAT'
+    $script:rg      = 'rg-devops-factory'
 
     Describe 'Get-AdfFromService' -Tag 'Unit' {
         It 'Should exist' {
             { Get-Command -Name Get-AdfFromService -ErrorAction Stop } | Should -Not -Throw
         }
-        
     } 
+
+    Describe 'Get-AdfFromService' -Tag 'Unit' {
+        It 'Should execute' {
+            Get-AdfFromService -FactoryName $adfName -ResourceGroupName $rg
+        }
+    } 
+
 }
diff --git a/test/Import-AdfFromFolder.Tests.ps1 b/test/Import-AdfFromFolder.Tests.ps1
@@ -128,7 +128,14 @@ InModuleScope azure.datafactory.tools {
             Context 'of adf2' {
                 It 'Should completed successfully' {
                     $script:RootFolder = "$PSScriptRoot\adf2"
-                    { Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $script:RootFolder -ErrorAction Stop } | Should -Not -Throw
+                    { $script:result = Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $script:RootFolder -ErrorAction Stop } | Should -Not -Throw
+                }
+                It 'Should contains Credentials prop as ArrayList and # of items matches' {
+                    $script:result.Credentials.GetType() | Should -Be 'System.Collections.ArrayList'
+                    $cnt = 0
+                    $folder = Join-Path -Path "$script:RootFolder" -ChildPath "Credential"
+                    if (Test-Path $folder) { $cnt = ($folder | Get-ChildItem -Recurse:$false -Filter "*.json" | Measure-Object).Count }
+                    $script:result.Credentials.Count | Should -Be $cnt
                 }
             }
 
@@ -147,6 +154,7 @@ InModuleScope azure.datafactory.tools {
                     Restore-File -FileName $bf $true
                     { Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $RootFolder -ErrorAction Stop } | Should -Not -Throw
                 }
+    
             }
 
         }
diff --git a/test/Integration.Tests.ps1 b/test/Integration.Tests.ps1
@@ -175,6 +175,22 @@ InModuleScope azure.datafactory.tools {
     } 
 
 
+    Describe 'Publish-AdfV2FromJson and DeployGlobalParams=true' -Tag 'Integration', 'global' {
+        Context 'But factory folder does not exist' {
+            It 'Should run successfully' {
+                $script:opt = New-AdfPublishOption
+                $script:opt.Excludes.Add("*.*", "")
+                $script:opt.DeployGlobalParams = $false
+                $script:opt.StopStartTriggers = $false
+                {
+                    Publish-AdfV2FromJson -RootFolder "$RootFolder" `
+                        -ResourceGroupName "$ResourceGroupName" `
+                        -DataFactoryName "$DataFactoryName" -Location "$Location" -Option $script:opt 
+                } | Should -Not -Throw
+            }
+        }
+    }
+
     Describe 'Publish-AdfV2FromJson and DeployGlobalParams=true' -Tag 'Integration', 'global' {
         Context 'But factory folder does not exist' {
             It 'Should run successfully' {
@@ -190,16 +206,28 @@ InModuleScope azure.datafactory.tools {
                 } | Should -Not -Throw
             }
         }
+
+        Context 'Change PublicEndpoint to Disabled' {
+            It 'Should be changed' {
+                $targetAdf = Get-AzDataFactoryV2 -ResourceGroupName $ResourceGroupName -Name $DataFactoryName
+                $df = Set-AzDataFactoryV2 -InputObject $targetAdf -Force -PublicNetworkAccess "Disabled"
+                $df.PublicNetworkAccess | Should -Be "Disabled"
+            }
+        }
+
         Context 'factory folder exists and some global params exist' {
             It 'Should run and deploy 7 global properties' {
                 Copy-Item -path "$SrcFolder" -Destination "$TmpFolder" -Filter "$($script:DataFactoryOrigName).json" -Recurse:$true -Force 
-                $adf = Import-AdfFromFolder -FactoryName "$($script:DataFactoryName)" -RootFolder "$RootFolder"
-                $gp = $adf.Factories[0].Body.properties.globalParameters
                 Publish-AdfV2FromJson -RootFolder "$RootFolder" `
                     -ResourceGroupName "$ResourceGroupName" `
                     -DataFactoryName "$DataFactoryName" -Location "$Location" -Option $script:opt 
-                $adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName"
-                $adfi.GlobalParameters.Count | Should -Be 7
+                    $script:adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -DataFactoryName "$DataFactoryName"
+                    $adfi.GlobalParameters.Count | Should -Be 7
+                }
+
+            It 'Should have all global params of right types' {
+                $adf = Import-AdfFromFolder -FactoryName "$($script:DataFactoryName)" -RootFolder "$RootFolder"
+                $gp = $adf.Factories[0].Body.properties.globalParameters
                 $adfi.GlobalParameters.'GP-String'.Type | Should -Be $gp.'GP-String'.type
                 $adfi.GlobalParameters.'GP-String'.Value | Should -Be $gp.'GP-String'.value
                 $adfi.GlobalParameters.'GP-Int'.Type | Should -Be $gp.'GP-Int'.type
diff --git a/test/Publish-AdfV2FromJson.Tests.ps1 b/test/Publish-AdfV2FromJson.Tests.ps1
@@ -11,11 +11,29 @@ InModuleScope azure.datafactory.tools {
     Import-Module -Name $testHelperPath -Force
 
     # Variables for use in tests
+    $script:ResourceGroupName = 'rg-devops-factory'
+    $c = Get-AzContext
+    $script:guid = $c.Subscription.Id.Substring(0,8)
+    $script:DataFactoryOrigName = 'adf2'
+    $script:DataFactoryName = $script:DataFactoryOrigName + "-$guid"
+    $script:Location = "UK South"
 
     Describe 'Publish-AdfV2FromJson' -Tag 'Unit' {
         It 'Should exist' {
             { Get-Command -Name Publish-AdfV2FromJson -ErrorAction Stop } | Should -Not -Throw
         }
-        
     } 
+
+    Describe 'Publish-AdfV2FromJson' {
+        It 'adf2 Should skip deployment of any credential object' {
+            $script:RootFolder = "$PSScriptRoot\adf2"
+            $o = New-AdfPublishOption
+            $o.StopStartTriggers = $false
+            $o.Includes.Add("cred*.*", "")
+            Publish-AdfV2FromJson -RootFolder $RootFolder -ResourceGroupName $script:ResourceGroupName `
+                -Location $script:Location -DataFactoryName $script:DataFactoryName -Option $o
+        }
+
+    }
+
 }
diff --git a/test/adf2/credential/credential1.json b/test/adf2/credential/credential1.json
@@ -0,0 +1,9 @@
+{
+	"name": "credential1",
+	"properties": {
+		"type": "ManagedIdentity",
+		"typeProperties": {
+			"resourceId": "/subscriptions/7d6cdb5f-be97-4907-a34c-66635a3f607d/resourceGroups/ADF.procfwk/providers/Microsoft.ManagedIdentity/userAssignedIdentities/iau-test1"
+		}
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`	`#`
`4`	`4`	`# Generated by: Kamil Nowinski @ SQLPlayer.net`
`5`	`5`	`#`
`6`		`-# Generated on: 25/04/2022`
	`6`	`+# Generated on: 10/10/2022`
`7`	`7`	`#`
`8`	`8`
`9`	`9`	`@{`
`@@ -12,7 +12,7 @@`
`12`	`12`	`RootModule = 'azure.datafactory.tools.psm1'`
`13`	`13`
`14`	`14`	`# Version number of this module.`
`15`		`- ModuleVersion = '0.97.0'`
	`15`	`+ ModuleVersion = '0.99.0'`
`16`	`16`
`17`	`17`	`# Supported PSEditions`
`18`	`18`	`# CompatiblePSEditions = @()`
Original file line number	Diff line number	Diff line change
`@@ -83,7 +83,7 @@ class AdfObject {`
`83`	`83`	`return $ofn`
`84`	`84`	`}`
`85`	`85`
`86`		`- static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory', 'managedVirtualNetwork', 'managedPrivateEndpoint')`
	`86`	`+ static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory', 'managedVirtualNetwork', 'managedPrivateEndpoint', 'credential')`
`87`	`87`
`88`	`88`	`static AssertType ([string] $Type)`
`89`	`89`	`{`
Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,10 @@ function Get-AdfObjectByName {`
`35`	`35`	`{`
`36`	`36`	`$r = $adf.Triggers \| Where-Object { $_.Name -eq $name } \| Select-Object -First 1`
`37`	`37`	`}`
	`38`	`+ 'Credential'`
	`39`	`+ {`
	`40`	`+ $r = $adf.Credentials \| Where-Object { $_.Name -eq $name } \| Select-Object -First 1`
	`41`	`+ }`
`38`	`42`	`'Factory'`
`39`	`43`	`{`
`40`	`44`	`$r = $adf.Factories \| Where-Object { $_.Name -eq $name } \| Select-Object -First 1`