Merge pull request #73 from SQLPlayer/develop

ver.0.19

Author: NowinskiK

README.md

@@ -217,14 +217,16 @@ Therefore you can use the following combinations:
 ```
 trigger.*
 dataset.DS_*
-*.PL_*
+*.PL_*@test*
 linkedService.???KeyVault*
 pipeline.ScdType[123]
+trigger.*@testFolder
 ```
-Full name of objects supported by the module is built of: `{Type}.{Name}`  
+Full name of objects supported by the module is built of: `{Type}.{Name}@{Folder}`  
 All potential combinations can be found in code repository of ADF:  
 *Type* - name of folder  
-*Name* - name of file (without JSON extension)
+*Name* - name of file (without JSON extension)  
+*Folder* - name of (ADF) folder to which objects belong to
 
 > More info about wildcard: [About Wildcard](https://docs.microsoft.com/en-gb/powershell/module/microsoft.powershell.core/about/about_wildcards?view=powershell-5.1)
 

azure.datafactory.tools.psd1

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.19.0] - 2020-12-23
+### Added
+* Support wildcard when specifying object(s) name in config file (#58)
+* Added option $IgnoreLackOfReferencedObject (#64)
+* Add object name to the msg before action (#49)
+* Exit publish cmd when ADF name is already in use (#43)
+* Allow selecting objects in given folder (#68)
+### Fixed
+* Finding dependencies miss objects when the same object names occurs (#65)
+* DeleteNotInSource fails when attempting to remove active trigger or found many dependant objects (#67)
 
 ## [0.18.0] - 2020-12-04
 ### Added

changelog.md

@@ -0,0 +1,19 @@
+# Messages:
+ADFTmxxx
+m = module/feature
+0 - Generic
+
+xxx - number of error
+
+## Index 
+ADFT0005: Referenced object [$name] was not found.
+ADFT0006: Referenced object [$name] was not found. No error raised as user wanted to carry on.
+
+ADFT0007: Could not find object: $type.$name
+ADFT0008: Could not find object: $type.$name
+ADFT0009: Body of the object is empty!
+ADFT0010: Wrong path defined in config for object(path): $type.$name(properties.$path)"
+ADFT0011: adf.Location property has not been provided.
+
+
+

en-us/messages_index.md

@@ -7,4 +7,5 @@ class AdfPublishOption {
     [Boolean] $DeployGlobalParams = $true
     [Boolean] $FailsWhenConfigItemNotFound = $true
     [Boolean] $FailsWhenPathNotFound = $true
+    [Boolean] $IgnoreLackOfReferencedObject = $false
 }

private/AdfPublishOption.class.ps1 private/!AdfPublishOption.class.ps1private/AdfPublishOption.class.ps1 renamed to private/!AdfPublishOption.class.ps1

@@ -17,6 +17,7 @@ class Adf {
     [System.Collections.ArrayList] $Factories = @{}
     [string] $Location = ""
     [AdfGlobalProp] $GlobalFactory = [AdfGlobalProp]::new()
+    [AdfPublishOption] $PublishOptions
 
     [System.Collections.ArrayList] AllObjects()
     {
@@ -49,5 +50,29 @@ class Adf {
         return $r
     }
 
+    [System.Collections.ArrayList] GetUnusedDatasets()
+    {
+        [System.Collections.ArrayList] $dataset_list = @{}
+        $this.DataSets | ForEach-Object { $null = $dataset_list.Add("$($_.Type.ToLower()).$($_.Name)") }
+
+        # Collect all objects used by pipelines and dataflows
+        $list = $this.Pipelines + $this.DataFlows
+        if ($list.Count -gt 0) { 
+            $list = $list.DependsOn
+
+            # Filter list to datasets only
+            $used = $list | Where-Object { $_.StartsWith('dataset.', "CurrentCultureIgnoreCase") } | `
+                            ForEach-Object { $_.Substring(8).Insert(0, 'dataset.') } | `
+                            Select-Object -Unique
+
+            # Remove all used datasets from $dataset_list
+            $used | ForEach-Object { $dataset_list.Remove($_) }
+        }
+
+        # datasets not removed from the list are the ones not being used
+        return $dataset_list
+    }   
+
+
 }
 

private/Adf.class.ps1

@@ -2,17 +2,18 @@ class AdfObject {
     [string] $Name
     [string] $Type
     [string] $FileName
-    [System.Collections.Hashtable] $DependsOn = @{}
+    [System.Collections.ArrayList] $DependsOn = @()
     [Boolean] $Deployed = $false
     [Boolean] $ToBeDeployed = $true
     [Adf] $Adf
     [PSCustomObject] $Body
 
-    [Boolean] AddDependant ([string]$name, [string]$type)
+    [Boolean] AddDependant ([string]$name, [string]$refType)
     {
-        $type2 = $type.Replace('Reference', '')
-        if (!$this.DependsOn.ContainsKey($name)) {
-            $this.DependsOn.Add( $name, $type2 ) | Out-Null
+        $objType = $refType.Replace('Reference', '')
+        $fullName = "$objType.$name"
+        if (!$this.DependsOn.Contains($fullName)) {
+            $this.DependsOn.Add( $fullName ) | Out-Null
         }
         return $true
     }
@@ -37,23 +38,40 @@ class AdfObject {
         return $this.FullName($true)
     }
 
+    [Boolean] IsNameMatch ([string]$wildcardPattern)
+    {
+        $folder = $this.GetFolderName()
+        $fullname = $this.FullName($false)
+        $arr = $wildcardPattern.Split('@')
+        $namePattern = $arr[0]
+        if ($arr.Count -le 1)
+        {
+            $r = ($fullname -like $namePattern) 
+        } else {
+            $folderPattern = $arr[1]
+            $r = ($fullname -like $namePattern) -and ( $folder -like $folderPattern )
+        }
+        return $r
+    }
+
     [String] GetFolderName()
     {
-        $ofn = $null
+        $ofn = ''
         if ($this.Body.PSObject.Properties.Name -contains "properties")
         {
             $o = $this.Body.properties
             if ($o.PSobject.Properties.Name -contains "folder")
             {
-                $ofn = $_.Body.properties.folder.name
+                $ofn = $this.Body.properties.folder.name
             }
         }
         return $ofn
     }
 
+    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory')
 
 }
 
 if (!(Get-Variable ADF_FOLDERS -ErrorAction:SilentlyContinue)) {
-    Set-Variable ADF_FOLDERS -option ReadOnly -value ('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory')
+    Set-Variable ADF_FOLDERS -option ReadOnly -value ([AdfObject]::AllowedTypes)
 }    

private/AdfObject.class.ps1

@@ -0,0 +1,25 @@
+class AdfObjectName {
+    [string] $Name
+    [string] $Type
+
+    AdfObjectName ([string] $Name, [string] $Type) 
+    {
+        $this.Name = $Name
+        $this.Type = $Type
+    }
+
+    AdfObjectName ([string] $FullName) 
+    {
+        if ($FullName.IndexOf('.') -lt 1) {
+            Write-Error "Expected format of name for 'FullName' input parameter is: objectType.objectName"
+        }
+        $parts = $FullName.Split('.')
+        if ($parts[0] -notin [AdfObject]::allowedTypes ) { 
+            Write-Error -Message "Unknown object type: $parts[0]."
+        }
+        $this.Type = $parts[0]
+        $this.Name = $parts[1]
+    }
+
+}
+

private/AdfObjectName.class.ps1

@@ -1,11 +1,11 @@
 function ApplyExclusionOptions {
     param(
-        [Parameter(Mandatory=$True)] [Adf] $adf,
-        [Parameter(Mandatory=$True)] [AdfPublishOption] $option
+        [Parameter(Mandatory=$True)] [Adf] $adf
     )
 
     Write-Debug "BEGIN: ApplyExclusionOptions()"
 
+    $option = $adf.PublishOptions
     if ($option.Excludes.Keys.Count -gt 0 -and $option.Includes.Keys.Count -eq 0)
     {
         Write-Debug "ENTRY: ApplyExclusionOptions()::Excludes"
@@ -17,8 +17,7 @@ function ApplyExclusionOptions {
             $key = $_
             $adf.AllObjects() | ForEach-Object {
                 [AdfObject] $o = $_
-                $nonDeployable = ($o.FullName($false) -like $key)
-                #Write-Debug "$($o.FullName($false)) -like $key"
+                $nonDeployable = $o.IsNameMatch($key)
                 if ($nonDeployable) { $o.ToBeDeployed = $false }
                 #Write-Verbose "- $($o.FullName($true)).ToBeDeployed = $($o.ToBeDeployed)"
             }
@@ -36,8 +35,7 @@ function ApplyExclusionOptions {
             $key = $_
             $adf.AllObjects() | ForEach-Object {
                 [AdfObject] $o = $_
-                $deployable = ($o.FullName($false) -like $key)
-                #Write-Debug "$($o.FullName($false)) -like $key"
+                $deployable = $o.IsNameMatch($key)
                 if ($deployable) { $o.ToBeDeployed = $true }
                 #Write-Verbose "- $($o.FullName($true)).ToBeDeployed = $($o.ToBeDeployed)"
             }

private/ApplyExclusionOptions.ps1

@@ -21,13 +21,18 @@ function Deploy-AdfObject {
     {
         Write-Debug "Checking all dependencies of [$($obj.Name)]..."
         $i = 1
-        $obj.DependsOn.getEnumerator() | ForEach-Object {
-            $name = $_.key
-            $type = $_.value
+        $obj.DependsOn | ForEach-Object {
+            $on = [AdfObjectName]::new($_)
+            $name = $on.Name
+            $type = $on.Type
             Write-Verbose ("$i) Depends on: [$type].[$name]")
             $depobj = Get-AdfObjectByName -adf $adf -name "$name" -type "$type"
             if ($null -eq $depobj) {
-                Write-Error "Referenced object [$name] was not found."
+                if ($adf.PublishOptions.IgnoreLackOfReferencedObject -eq $true) {
+                    Write-Warning "ADFT0006: Referenced object [$type].[$name] was not found. No error raised as user wanted to carry on."
+                } else {
+                    Write-Error "ADFT0005: Referenced object [$type].[$name] was not found."
+                }
             } else {
                 Deploy-AdfObject -obj $depobj
             }