Skip to content

Commit 1218924

Browse files
NowinskiKNowinskiK
authored
Merge pull request #151 from dataplat/feature/143-combine
Combine Invoke-FabricRestMethod and Test-FabricApiResponse
2 parents 0abbd41 + 9516b81 commit 1218924

23 files changed

+395
-475
lines changed

‎CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
2323
- `Remove-FabricWorkspaceFromStage`,
2424
- `Start-FabricDeploymentPipelineStage`
2525
- Added private function `Get-FabricContinuationToken` to facilitate pagination
26+
- `Invoke-FabricRestMethod` handles throttling (error 429) by pausing and repeating the request (#88)
2627

2728
### Changed
2829

@@ -32,6 +33,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
3233
- `Get-FabricSqlDatabase` accepts Workspace as a pipeline, handles errors correctly and can filter by name (#117).
3334
- Applied splatting for several parameters in `Invoke-FabricRestMethod` and output results in debug mode
3435
- `Remove-FabricSQLDatabase` uses unified function to handle API results
36+
- Internal function `Invoke-FabricRestMethod`: (#143)
37+
- handles API response, no need to use `Test-FabricApiResponse` from parent public function
38+
- handles pagination automatically (when `-HandleResponse` is provided)
39+
- All Deployment Pipeline functions raise an error when an exception is caught. Used splatting for params.
40+
- Refactored SQL Database functions to use enhanced capability in `Invoke-FabricRestMethod`. Used splatting for params.
41+
- `Write-Message` uses PSFramework function for logging, which logs function name (#84)
3542

3643
Updated the `WorkspaceId`, `CapacitiesIds`,`CapacityId`,`CopyJobId`,`datamartId`,`DataPipelineId`,`DataWarehouseGUID`,`DomainId`,`EnvironmentId`,`EventhouseId`,`EventstreamId`,`ExternalDataShareId`,`ItemId`,`KQLDashboardId`,`KQLDatabaseId`,`KQLQuerysetId`,`LakehouseId`,`MirroredDatabaseId`,`MirroredWarehouseId`,`MLExperimentId`,`MLModelId`,`NotebookId`,`operationId`,`PaginatedReportId`,`ParentDomainId`,`parentEventhouseId`,`PrincipalId`,`ReflexId`,`ReportId`,`SemanticModelId`,`SparkCustomPoolId`,`SparkJobDefinitionId`,`SQLDatabaseId`,`SQLEndpointId`,`subscriptionID`,`UserId`,`WarehouseId`,`WorkspaceGUID`,`WorkspaceId`,`WorkspaceIds`,and `WorkspaceRoleAssignmentId` parameters to the datatype GUID [#125](https://github.com/dataplat/FabricTools/issues/125)
3744

@@ -40,6 +47,7 @@ Updated the `WorkspaceId`, `CapacitiesIds`,`CapacityId`,`CopyJobId`,`datamartId`
4047
- Enhanced logic in unified function `Test-FabricApiResponse` to handle API results and moved it to private functions
4148
- Fixed bug in `Get-FabricLongRunningOperation` - Uri was incorectly created (#131)
4249
- Fixed bug in `Get-FabricLongRunningOperationResult` - uses correct statusCode (#131)
50+
- Fixed `Start-FabricDeploymentPipelineStage` that supports `-NoWait` correctly
4351

4452
### Deprecated
4553

‎helper/build-and-test.ps1

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ Invoke-ScriptAnalyzer -Path .\source\Public\**
2121
$tests = Invoke-Pester .\tests\ -PassThru
2222
$tests.Tests | where Result -eq 'Failed' | Measure-Object | Select-Object -ExpandProperty Count
2323
$tests.Tests | where Result -eq 'Failed' | ft -Property ExpandedName, ErrorRecord
24-
$tests.Tests | where Result -eq 'Failed' | ft -Property Name, Result, ErrorRecord -AutoSize
24+
$tests.Tests | where Result -eq 'Failed' | ft -Property Path, Result, ErrorRecord -AutoSize
2525

26-
$e = $tests.Tests | where Result -eq 'Failed' | Select-Object -First 1
26+
$e = $tests.Tests | where Result -eq 'Failed' | Select-Object -Last 1
2727
$e.ErrorRecord

‎source/Private/Get-FabricContinuationToken.ps1

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,14 @@ Author: Kamil Nowinski
2323
function Get-FabricContinuationToken {
2424
[CmdletBinding()]
2525
param (
26-
[Parameter(Mandatory = $true)]
26+
[Parameter(Mandatory = $false)]
2727
[object]$Response
2828
)
2929

3030
$continuationToken = $null
3131
if ($null -ne $Response) {
3232
# Update the continuation token if present
33-
if ($Response.PSObject.Properties.Match("continuationToken")) {
33+
if ($Response.PSObject.Properties.Match("continuationToken") -and $Response.continuationToken -ne $null) {
3434
$continuationToken = $Response.continuationToken
3535
Write-Message -Message "New continuation token: $continuationToken" -Level Debug
3636
} else {

‎source/Private/Test-FabricApiResponse.ps1

Lines changed: 70 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,26 @@ function Test-FabricApiResponse {
1010
.PARAMETER Response
1111
The response body from the API call.
1212
13+
.PARAMETER ResponseHeader
14+
The response headers from the API call. This is used to retrieve operation IDs and other metadata.
15+
16+
.PARAMETER StatusCode
17+
The HTTP status code returned by the API call. This is used to determine how to handle the response.
18+
19+
.PARAMETER Operation
20+
The operation being performed by parent function (e.g., 'New', 'Update', 'Remove', 'Get'). It helps in logging appropriate messages.
21+
This parameter is optional and can be used to customize the logging message based on the operation type.
22+
1323
.PARAMETER ObjectIdOrName
1424
The name or ID of the resource being operated.
1525
1626
.PARAMETER TypeName
1727
The type of resource being operated (default: 'Fabric Item').
1828
29+
.PARAMETER SuccessMessage
30+
A custom success message to log upon successful completion of the operation. This overrides the default message based on Operation, TypeName, ObjectIdOrName.
31+
This parameter is optional and can be used to provide a specific success message for the operation.
32+
1933
.PARAMETER NoWait
2034
If specified, the function will not wait for the operation to complete and will return immediately.
2135
@@ -33,48 +47,61 @@ function Test-FabricApiResponse {
3347
- This function is designed to be used within the context of a Fabric API client.
3448
- It requires the `Write-Message` function to log messages at different levels (Info, Debug, Error).
3549
- The function handles long-running operations by checking the status of the operation and retrieving the result if it has succeeded.
50+
- Supports handling of API rate limiting (HTTP 429) by waiting for the specified retry duration before returning a command to repeat the request.
3651
3752
Author: Kamil Nowinski
3853
3954
#>
4055

4156
[CmdletBinding()]
4257
param (
43-
[Parameter(Mandatory = $false)]
58+
[Parameter(Mandatory = $false)] # Response is $null when Response Code is 202
4459
$Response,
4560

61+
[Parameter(Mandatory = $true)]
62+
$ResponseHeader,
63+
64+
[Parameter(Mandatory = $true)]
65+
$StatusCode,
66+
67+
[Parameter(Mandatory = $false)]
68+
$Operation,
69+
4670
[Parameter(Mandatory = $false)]
4771
[string] $ObjectIdOrName,
4872

4973
[Parameter(Mandatory = $false)]
5074
[string] $TypeName = 'Fabric Item',
5175

76+
[Parameter(Mandatory = $false)]
77+
[string] $SuccessMessage,
78+
5279
[Parameter(Mandatory = $false)]
5380
[switch] $NoWait = $false
5481
)
5582

56-
Write-Message -Message "[Test-FabricApiResponse]::Begin" -Level Debug
83+
Write-Message -Message "::Begin" -Level Debug
5784

58-
$responseHeader = $script:responseHeader
59-
$statusCode = $script:statusCode
85+
#$responseHeader = $script:responseHeader
86+
#$statusCode = $script:statusCode
6087
$result = $null
61-
62-
$verb = (Get-PSCallStack)[1].Command.Split('-')[0]
63-
Write-Message -Message "Testing API response for '$verb' operation. StatusCode: $statusCode." -Level Debug
88+
Write-Message -Message "Testing API response for '$Operation' operation. StatusCode: $statusCode." -Level Debug
6489

6590
switch ($statusCode) {
6691
200 {
67-
$result = $null
92+
if ($Operation -eq 'Get') {
93+
$result = $Response
94+
}
6895
}
6996
201 {
7097
$result = $Response
7198
}
7299
202 {
73-
Write-Message -Message "$verb Request for $TypeName '$ObjectIdOrName' accepted. Provisioning in progress!" -Level Info
100+
Write-Message -Message "$Operation Request for $TypeName '$ObjectIdOrName' accepted. Provisioning in progress!" -Level Info
74101
[string]$operationId = $responseHeader["x-ms-operation-id"]
75102

76103
if ($NoWait) {
77-
Write-Message -Message "NoWait parameter is set. Operation ID: $operationId" -Level Info
104+
Write-Message -Message "NoWait parameter is set. Operation ID: $operationId" -Level Debug
78105
Write-Message -Message "Run to check the progress: Get-FabricLongRunningOperationResult -operationId '$operationId'" -Level Verbose
79106
return [PSCustomObject]@{
80107
Location = $responseHeader["Location"]
@@ -83,26 +110,35 @@ function Test-FabricApiResponse {
83110
}
84111
}
85112

86-
Write-Message -Message "[Test-FabricApiResponse] Operation ID: '$operationId'" -Level Debug
87-
Write-Message -Message "[Test-FabricApiResponse] Getting Long Running Operation status" -Level Debug
113+
Write-Message -Message "Operation ID: '$operationId'" -Level Debug
114+
Write-Message -Message "Getting Long Running Operation status" -Level Debug
88115

89116
$operationStatus = Get-FabricLongRunningOperation -operationId $operationId
90-
Write-Message -Message "[Test-FabricApiResponse] Long Running Operation status: $operationStatus" -Level Debug
117+
Write-Message -Message "Long Running Operation status: $operationStatus" -Level Debug
91118
# Handle operation result
92119
if ($operationStatus.status -eq "Succeeded") {
93120
Write-Message -Message "Operation Succeeded" -Level Debug
94-
Write-Message -Message "Getting Long Running Operation result" -Level Debug
121+
Write-Message -Message "Getting Long Running Operation result" -Level Verbose
95122

96123
$operationResult = Get-FabricLongRunningOperationResult -operationId $operationId
97-
Write-Message -Message "Long Running Operation status: $operationResult" -Level Debug
124+
#Write-Message -Message "Long Running Operation status: $operationResult" -Level Debug
98125

99126
return $operationResult
100127
} else {
101-
Write-Message -Message "Operation failed. Status: $($operationStatus)" -Level Debug
128+
#Write-Message -Message "Operation failed. Status: $($operationStatus)" -Level Debug
102129
Write-Message -Message "Operation failed. Status: $($operationStatus)" -Level Error
103130
return $operationStatus
104131
}
105132
}
133+
429 {
134+
Write-Message -Message "API rate limit exceeded. Status Code: $statusCode ($($Response.errorCode))" -Level Warning
135+
Write-Message -Message "$($Response.message). Waiting $($responseHeader['Retry-After']) seconds..." -Level Warning
136+
$retryAfter = $responseHeader['Retry-After']
137+
$retryAfterStr = $retryAfter[0].ToString()
138+
$retryAfterInt = [int]$retryAfterStr
139+
Start-Sleep -Seconds $retryAfterInt
140+
return "Command:Repeat"
141+
}
106142
default {
107143
Write-Message -Message "Test-FabricApiResponse::default" -Level Debug
108144
Write-Message -Message "Unexpected response code: $statusCode from the API." -Level Error
@@ -115,16 +151,29 @@ function Test-FabricApiResponse {
115151
}
116152
}
117153

118-
switch ($verb) {
154+
# if (FeatureFlag.IsEnabled('FabricToolsVerboseLogging')) { # This is placeholder for verbose logging feature flag being implemented soon
155+
$TypeName= $TypeName.Substring(0, 1).ToUpper() + $TypeName.Substring(1)
156+
157+
if ($SuccessMessage) {
158+
$Operation = "Custom"
159+
}
160+
161+
# Try to get Name of Id from the response when new item is created
162+
if ($Operation -eq 'New' -and -not $ObjectIdOrName) {
163+
$ObjectIdOrName = $Response.DisplayName ? $Response.DisplayName : $Response.id
164+
}
165+
switch ($Operation) {
119166
'New' { $msg = "$TypeName '$ObjectIdOrName' created successfully."; $level = 'Info' }
120167
'Update' { $msg = "$TypeName '$ObjectIdOrName' updated successfully."; $level = 'Info' }
121168
'Remove' { $msg = "$TypeName '$ObjectIdOrName' deleted successfully."; $level = 'Info' }
122-
'Get' { $msg = "Successfully retrieved $TypeName details."; $level = 'Debug' }
123-
default { $msg = "Received $statusCode status code for $verb operation on $TypeName '$ObjectIdOrName'."; $level = 'Info' }
169+
'Get' { $msg = "Successfully retrieved $TypeName details."; $level = 'Debug' }
170+
'Custom' { $msg = "$SuccessMessage"; $level = 'Info' }
171+
default { $msg = "Received $statusCode status code for $Operation operation on $TypeName '$ObjectIdOrName'."; $level = 'Info' }
124172
}
125173
Write-Message -Message $msg -Level $level
126-
Write-Message -Message "[Test-FabricApiResponse]::End" -Level Debug
174+
# }
127175

128-
$result
176+
Write-Message -Message "::End" -Level Debug
129177

178+
$result
130179
}

‎source/Private/Write-Message.ps1

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -54,17 +54,18 @@ function Write-Message {
5454
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
5555

5656
# Construct log message
57-
$logMessage = "[$timestamp] [$Level] $Message"
57+
$logMessage = $Message
58+
$fn = (Get-PSCallStack)[1].Command
5859

5960
# Write log message to console with colors
6061
switch ($Level) {
61-
"Message" { Write-Host $logMessage -ForegroundColor White }
62-
"Info" { Write-Host $logMessage -ForegroundColor Green }
63-
"Error" { Write-Error $logMessage }
64-
"Warning" { Write-Warning $logMessage }
65-
"Critical" { Write-Host $logMessage -ForegroundColor Red } # Or maybe Stop-PSFunction here?
66-
"Verbose" { Write-Verbose $logMessage }
67-
"Debug" { Write-Debug $logMessage }
62+
"Message" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Host }
63+
"Info" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Important }
64+
"Error" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Error }
65+
"Warning" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Warning }
66+
"Critical" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Critical }
67+
"Verbose" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Verbose }
68+
"Debug" { Write-PSFMessage -Message $logMessage -FunctionName $fn -Level Debug }
6869
}
6970

7071
# Optionally write log message to a file
@@ -77,7 +78,7 @@ function Write-Message {
7778
}
7879
}
7980
} catch {
80-
Write-Host "[ERROR] An unexpected error occurred: $_" -ForegroundColor Red
81+
Stop-PSFFunction -Message "An unexpected error occurred while writing the message to log file." -ErrorRecord $_ -Level Error
8182
}
8283
}
8384
}

‎source/Public/Deployment Pipeline/Add-FabricWorkspaceToStage.ps1

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,15 +64,23 @@ function Add-FabricWorkspaceToStage {
6464
}
6565

6666
# Step 4: Make the API request and validate response
67-
$response = Invoke-FabricRestMethod -Uri $apiEndpointUrl -Method Post -Body $requestBody
68-
Test-FabricApiResponse -Response $response
67+
$apiParameters = @{
68+
Uri = $apiEndpointUrl
69+
Method = 'POST'
70+
Body = $requestBody
71+
HandleResponse = $true
72+
TypeName = "deployment pipeline stage"
73+
ObjectIdOrName = $StageId
74+
SuccessMessage = "Successfully assigned workspace to deployment pipeline stage."
75+
}
76+
$response = Invoke-FabricRestMethod @apiParameters
6977

7078
# Step 5: Return results
71-
Write-Message -Message "Successfully assigned workspace to deployment pipeline stage." -Level Info
7279
$response
80+
7381
} catch {
7482
# Step 6: Error handling
7583
$errorDetails = $_.Exception.Message
76-
Write-Message -Message "Failed to assign workspace to deployment pipeline stage. Error: $errorDetails" -Level Error
84+
Write-Error -Message "Failed to assign workspace to deployment pipeline stage. Error: $errorDetails"
7785
}
7886
}

‎source/Public/Deployment Pipeline/Get-FabricDeploymentPipeline.ps1

Lines changed: 12 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -58,80 +58,32 @@ function Get-FabricDeploymentPipeline {
5858
if ($DeploymentPipelineId) {
5959
Write-Message -Message "Retrieving specific deployment pipeline with ID: $DeploymentPipelineId" -Level Debug
6060
$apiEndpointUrl = "deploymentPipelines/$DeploymentPipelineId"
61-
62-
$response = Invoke-FabricRestMethod -Uri $apiEndpointUrl -Method Get
63-
64-
# Validate response
65-
Test-FabricApiResponse -response $response -ObjectIdOrName $DeploymentPipelineId -typeName "deployment pipeline"
66-
67-
if ($response) {
68-
Write-Message -Message "Successfully retrieved deployment pipeline." -Level Debug
69-
return $response
70-
} else {
71-
Write-Message -Message "No deployment pipeline found with the specified ID." -Level Warning
72-
return $null
73-
}
61+
} else {
62+
$apiEndpointUrl = "deploymentPipelines"
7463
}
7564

76-
# Step 2: Initialize variables for listing all pipelines
77-
$continuationToken = $null
78-
$pipelines = @()
79-
80-
if (-not ([AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq "System.Web" })) {
81-
Add-Type -AssemblyName System.Web
65+
# Step 5: Make the API request
66+
$apiParameters = @{
67+
Uri = $apiEndpointUrl
68+
Method = 'GET'
69+
HandleResponse = $true
70+
TypeName = "deployment pipeline"
8271
}
83-
84-
# Step 3: Loop to retrieve all pipelines with continuation token
85-
Write-Message -Message "Loop started to get continuation token" -Level Debug
86-
$baseApiEndpointUrl = "deploymentPipelines"
87-
88-
do {
89-
# Step 4: Construct the API URL
90-
$apiEndpointUrl = $baseApiEndpointUrl
91-
92-
if ($null -ne $continuationToken) {
93-
# URL-encode the continuation token
94-
$encodedToken = [System.Web.HttpUtility]::UrlEncode($continuationToken)
95-
$apiEndpointUrl = "{0}?continuationToken={1}" -f $apiEndpointUrl, $encodedToken
96-
}
97-
Write-Message -Message "API Endpoint: $apiEndpointUrl" -Level Debug
98-
99-
# Step 5: Make the API request
100-
$response = Invoke-FabricRestMethod -Uri $apiEndpointUrl -Method Get
101-
102-
# Validate response
103-
Test-FabricApiResponse -response $response -typeName "deployment pipeline"
104-
105-
# Step 6: Process response and update continuation token
106-
if ($null -ne $response) {
107-
$pipelines += $response.value
108-
}
109-
$continuationToken = Get-FabricContinuationToken -Response $response
110-
111-
} while ($null -ne $continuationToken)
112-
113-
Write-Message -Message "Loop finished and all data added to the list" -Level Debug
72+
$response = Invoke-FabricRestMethod @apiParameters
11473

11574
if ($DeploymentPipelineName)
11675
{
11776
# Filter the list by name
11877
Write-Message -Message "Filtering deployment pipelines by name: $DeploymentPipelineName" -Level Debug
119-
$pipelines = $pipelines | Where-Object { $_.displayName -eq $DeploymentPipelineName }
78+
$response = $response | Where-Object { $_.displayName -eq $DeploymentPipelineName }
12079
}
12180

12281
# Step 7: Handle results
123-
$pipelines
124-
# if ($pipelines) {
125-
# Write-Message -Message "Successfully retrieved deployment pipelines." -Level Debug
126-
# return $pipelines
127-
# } else {
128-
# Write-Message -Message "No deployment pipelines found." -Level Warning
129-
# return $null
130-
# }
82+
$response
13183

13284
} catch {
13385
# Step 8: Error handling
13486
$errorDetails = $_.Exception.Message
135-
Write-Message -Message "Failed to retrieve deployment pipelines. Error: $errorDetails" -Level Error
87+
Write-Error -Message "Failed to retrieve deployment pipelines. Error: $errorDetails"
13688
}
13789
}

0 commit comments

Comments
 (0)