Skip to content

Commit de77f61

Browse files
JohnDupreyJohnDuprey
committed
Refactor backup to use blob storage and enhance restore
Backups are now stored as blobs in Azure Storage with table entities referencing the blob URLs, improving scalability and performance. The backup listing, creation, and retention cleanup functions have been updated to handle blob-based backups, including proper cleanup of both blob files and table entries. Restore logic is enhanced to fetch and parse blob content, and restoration tasks now provide more detailed feedback and error handling. These changes modernize the backup/restore pipeline and improve reliability for large backup data.
1 parent 7aa4490 commit de77f61

File tree

8 files changed

+610
-204
lines changed

8 files changed

+610
-204
lines changed

‎Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecListBackup.ps1‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,22 @@ function Invoke-ExecListBackup {
3131
Items = $properties.Name
3232
}
3333
} else {
34+
# Prefer stored indicator (BackupIsBlob) to avoid reading Backup field
35+
$isBlob = $false
36+
if ($null -ne $item.PSObject.Properties['BackupIsBlob']) {
37+
try { $isBlob = [bool]$item.BackupIsBlob } catch { $isBlob = $false }
38+
} else {
39+
# Fallback heuristic for legacy rows if property missing
40+
if ($null -ne $item.PSObject.Properties['Backup']) {
41+
$b = $item.Backup
42+
if ($b -is [string] -and ($b -like 'https://*' -or $b -like 'http://*')) { $isBlob = $true }
43+
}
44+
}
3445
[PSCustomObject]@{
3546
BackupName = $item.RowKey
3647
Timestamp = $item.Timestamp
48+
Source = if ($isBlob) { 'blob' } else { 'table' }
3749
}
38-
3950
}
4051
}
4152
$Result = $Processed | Sort-Object Timestamp -Descending

‎Modules/CIPPCore/Public/Entrypoints/HTTP Functions/CIPP/Settings/Invoke-ExecRestoreBackup.ps1‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,26 @@ function Invoke-ExecRestoreBackup {
1212
try {
1313

1414
if ($Request.Body.BackupName -like 'CippBackup_*') {
15-
$Table = Get-CippTable -tablename 'CIPPBackup'
16-
$Backup = Get-CippAzDataTableEntity @Table -Filter "RowKey eq '$($Request.Body.BackupName)' or OriginalEntityId eq '$($Request.Body.BackupName)'"
15+
# Use Get-CIPPBackup which already handles fetching from blob storage
16+
$Backup = Get-CIPPBackup -Type 'CIPP' -Name $Request.Body.BackupName
1717
if ($Backup) {
18-
$BackupData = $Backup.Backup | ConvertFrom-Json -ErrorAction SilentlyContinue | Select-Object * -ExcludeProperty ETag, Timestamp
18+
$raw = $Backup.Backup
19+
$BackupData = $null
20+
21+
# Get-CIPPBackup already fetches blob content, so raw should be JSON string
22+
try {
23+
if ($raw -is [string]) {
24+
$BackupData = $raw | ConvertFrom-Json -ErrorAction Stop
25+
} else {
26+
$BackupData = $raw | Select-Object * -ExcludeProperty ETag, Timestamp
27+
}
28+
} catch {
29+
throw "Failed to parse backup JSON: $($_.Exception.Message)"
30+
}
31+
1932
$BackupData | ForEach-Object {
2033
$Table = Get-CippTable -tablename $_.table
21-
$ht2 = @{ }
34+
$ht2 = @{}
2235
$_.psobject.properties | ForEach-Object { $ht2[$_.Name] = [string]$_.Value }
2336
$Table.Entity = $ht2
2437
Add-CIPPAzDataTableEntity @Table -Force

‎Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-BackupRetentionCleanup.ps1‎

Lines changed: 112 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ function Start-BackupRetentionCleanup {
1313
$ConfigTable = Get-CippTable -tablename Config
1414
$Filter = "PartitionKey eq 'BackupRetention' and RowKey eq 'Settings'"
1515
$RetentionSettings = Get-CIPPAzDataTableEntity @ConfigTable -Filter $Filter
16-
16+
1717
# Default to 30 days if not set
18-
$RetentionDays = if ($RetentionSettings.RetentionDays) {
19-
[int]$RetentionSettings.RetentionDays
20-
} else {
21-
30
18+
$RetentionDays = if ($RetentionSettings.RetentionDays) {
19+
[int]$RetentionSettings.RetentionDays
20+
} else {
21+
30
2222
}
2323

2424
# Ensure minimum retention of 7 days
@@ -27,24 +27,67 @@ function Start-BackupRetentionCleanup {
2727
}
2828

2929
Write-Host "Starting backup cleanup with retention of $RetentionDays days"
30-
30+
3131
# Calculate cutoff date
3232
$CutoffDate = (Get-Date).AddDays(-$RetentionDays).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
33-
33+
3434
$DeletedCounts = [System.Collections.Generic.List[int]]::new()
3535

3636
# Clean up CIPP Backups
3737
if ($PSCmdlet.ShouldProcess('CIPPBackup', 'Cleaning up old backups')) {
3838
$CIPPBackupTable = Get-CippTable -tablename 'CIPPBackup'
39-
$Filter = "PartitionKey eq 'CIPPBackup' and Timestamp lt datetime'$CutoffDate'"
40-
41-
$OldCIPPBackups = Get-AzDataTableEntity @CIPPBackupTable -Filter $Filter -Property @('PartitionKey', 'RowKey', 'ETag')
42-
43-
if ($OldCIPPBackups) {
44-
Write-Host "Found $($OldCIPPBackups.Count) old CIPP backups to delete"
45-
Remove-AzDataTableEntity @CIPPBackupTable -Entity $OldCIPPBackups -Force
46-
$DeletedCounts.Add($OldCIPPBackups.Count)
47-
Write-LogMessage -API 'BackupRetentionCleanup' -message "Deleted $($OldCIPPBackups.Count) old CIPP backups" -Sev 'Info'
39+
$CutoffFilter = "PartitionKey eq 'CIPPBackup' and Timestamp lt datetime'$CutoffDate'"
40+
41+
# Delete blob files
42+
$BlobFilter = "$CutoffFilter and BackupIsBlob eq true"
43+
$BlobBackups = Get-AzDataTableEntity @CIPPBackupTable -Filter $BlobFilter -Property @('PartitionKey', 'RowKey', 'Backup')
44+
45+
$BlobDeletedCount = 0
46+
if ($BlobBackups) {
47+
foreach ($Backup in $BlobBackups) {
48+
if ($Backup.Backup) {
49+
try {
50+
$BlobPath = $Backup.Backup
51+
# Extract container/blob path from URL
52+
if ($BlobPath -like '*:10000/*') {
53+
# Azurite format: http://host:10000/devstoreaccount1/container/blob
54+
$parts = $BlobPath -split ':10000/'
55+
if ($parts.Count -gt 1) {
56+
# Remove account name to get container/blob
57+
$BlobPath = ($parts[1] -split '/', 2)[-1]
58+
}
59+
} elseif ($BlobPath -like '*blob.core.windows.net/*') {
60+
# Azure Storage format: https://account.blob.core.windows.net/container/blob
61+
$BlobPath = ($BlobPath -split '.blob.core.windows.net/', 2)[-1]
62+
}
63+
$null = New-CIPPAzStorageRequest -Service 'blob' -Resource $BlobPath -Method 'DELETE' -ConnectionString $ConnectionString
64+
$BlobDeletedCount++
65+
Write-Host "Deleted blob: $BlobPath"
66+
} catch {
67+
Write-LogMessage -API 'BackupRetentionCleanup' -message "Failed to delete blob $($Backup.Backup): $($_.Exception.Message)" -Sev 'Warning'
68+
}
69+
}
70+
}
71+
# Delete blob table entities
72+
Remove-AzDataTableEntity @CIPPBackupTable -Entity $BlobBackups -Force
73+
}
74+
75+
# Delete table-only backups (no blobs)
76+
# Fetch all old entries and filter out blob entries client-side (null check is unreliable in filters)
77+
$AllOldBackups = Get-AzDataTableEntity @CIPPBackupTable -Filter $CutoffFilter -Property @('PartitionKey', 'RowKey', 'ETag', 'BackupIsBlob')
78+
$TableBackups = $AllOldBackups | Where-Object { $_.BackupIsBlob -ne $true }
79+
80+
$TableDeletedCount = 0
81+
if ($TableBackups) {
82+
Remove-AzDataTableEntity @CIPPBackupTable -Entity $TableBackups -Force
83+
$TableDeletedCount = ($TableBackups | Measure-Object).Count
84+
}
85+
86+
$TotalDeleted = $BlobDeletedCount + $TableDeletedCount
87+
if ($TotalDeleted -gt 0) {
88+
$DeletedCounts.Add($TotalDeleted)
89+
Write-LogMessage -API 'BackupRetentionCleanup' -message "Deleted $TotalDeleted old CIPP backups ($BlobDeletedCount blobs, $TableDeletedCount table entries)" -Sev 'Info'
90+
Write-Host "Deleted $TotalDeleted old CIPP backups"
4891
} else {
4992
Write-Host 'No old CIPP backups found'
5093
}
@@ -53,23 +96,66 @@ function Start-BackupRetentionCleanup {
5396
# Clean up Scheduled/Tenant Backups
5497
if ($PSCmdlet.ShouldProcess('ScheduledBackup', 'Cleaning up old backups')) {
5598
$ScheduledBackupTable = Get-CippTable -tablename 'ScheduledBackup'
56-
$Filter = "PartitionKey eq 'ScheduledBackup' and Timestamp lt datetime'$CutoffDate'"
57-
58-
$OldScheduledBackups = Get-AzDataTableEntity @ScheduledBackupTable -Filter $Filter -Property @('PartitionKey', 'RowKey', 'ETag')
59-
60-
if ($OldScheduledBackups) {
61-
Write-Host "Found $($OldScheduledBackups.Count) old tenant backups to delete"
62-
Remove-AzDataTableEntity @ScheduledBackupTable -Entity $OldScheduledBackups -Force
63-
$DeletedCounts.Add($OldScheduledBackups.Count)
64-
Write-LogMessage -API 'BackupRetentionCleanup' -message "Deleted $($OldScheduledBackups.Count) old tenant backups" -Sev 'Info'
99+
$CutoffFilter = "PartitionKey eq 'ScheduledBackup' and Timestamp lt datetime'$CutoffDate'"
100+
101+
# Delete blob files
102+
$BlobFilter = "$CutoffFilter and BackupIsBlob eq true"
103+
$BlobBackups = Get-AzDataTableEntity @ScheduledBackupTable -Filter $BlobFilter -Property @('PartitionKey', 'RowKey', 'Backup')
104+
105+
$BlobDeletedCount = 0
106+
if ($BlobBackups) {
107+
foreach ($Backup in $BlobBackups) {
108+
if ($Backup.Backup) {
109+
try {
110+
$BlobPath = $Backup.Backup
111+
# Extract container/blob path from URL
112+
if ($BlobPath -like '*:10000/*') {
113+
# Azurite format: http://host:10000/devstoreaccount1/container/blob
114+
$parts = $BlobPath -split ':10000/'
115+
if ($parts.Count -gt 1) {
116+
# Remove account name to get container/blob
117+
$BlobPath = ($parts[1] -split '/', 2)[-1]
118+
}
119+
} elseif ($BlobPath -like '*blob.core.windows.net/*') {
120+
# Azure Storage format: https://account.blob.core.windows.net/container/blob
121+
$BlobPath = ($BlobPath -split '.blob.core.windows.net/', 2)[-1]
122+
}
123+
$null = New-CIPPAzStorageRequest -Service 'blob' -Resource $BlobPath -Method 'DELETE' -ConnectionString $ConnectionString
124+
$BlobDeletedCount++
125+
Write-Host "Deleted blob: $BlobPath"
126+
} catch {
127+
Write-LogMessage -API 'BackupRetentionCleanup' -message "Failed to delete blob $($Backup.Backup): $($_.Exception.Message)" -Sev 'Warning'
128+
}
129+
}
130+
}
131+
# Delete blob table entities
132+
Remove-AzDataTableEntity @ScheduledBackupTable -Entity $BlobBackups -Force
133+
}
134+
135+
# Delete table-only backups (no blobs)
136+
# Fetch all old entries and filter out blob entries client-side (null check is unreliable in filters)
137+
$AllOldBackups = Get-AzDataTableEntity @ScheduledBackupTable -Filter $CutoffFilter -Property @('PartitionKey', 'RowKey', 'ETag', 'BackupIsBlob')
138+
$TableBackups = $AllOldBackups | Where-Object { $_.BackupIsBlob -ne $true }
139+
140+
$TableDeletedCount = 0
141+
if ($TableBackups) {
142+
Remove-AzDataTableEntity @ScheduledBackupTable -Entity $TableBackups -Force
143+
$TableDeletedCount = ($TableBackups | Measure-Object).Count
144+
}
145+
146+
$TotalDeleted = $BlobDeletedCount + $TableDeletedCount
147+
if ($TotalDeleted -gt 0) {
148+
$DeletedCounts.Add($TotalDeleted)
149+
Write-LogMessage -API 'BackupRetentionCleanup' -message "Deleted $TotalDeleted old tenant backups ($BlobDeletedCount blobs, $TableDeletedCount table entries)" -Sev 'Info'
150+
Write-Host "Deleted $TotalDeleted old tenant backups"
65151
} else {
66152
Write-Host 'No old tenant backups found'
67153
}
68154
}
69155

70156
$TotalDeleted = ($DeletedCounts | Measure-Object -Sum).Sum
71157
Write-LogMessage -API 'BackupRetentionCleanup' -message "Backup cleanup completed. Total backups deleted: $TotalDeleted (retention: $RetentionDays days)" -Sev 'Info'
72-
158+
73159
} catch {
74160
$ErrorMessage = Get-CippException -Exception $_
75161
Write-LogMessage -API 'BackupRetentionCleanup' -message "Failed to run backup cleanup: $($ErrorMessage.NormalizedError)" -Sev 'Error' -LogData $ErrorMessage

‎Modules/CIPPCore/Public/Get-CIPPBackup.ps1‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,15 @@ function Get-CIPPBackup {
1818
}
1919

2020
if ($NameOnly.IsPresent) {
21-
$Table.Property = @('RowKey')
21+
if ($Type -ne 'Scheduled') {
22+
$Table.Property = @('RowKey', 'Timestamp', 'BackupIsBlob')
23+
} else {
24+
$Table.Property = @('RowKey', 'Timestamp')
25+
}
26+
}
27+
28+
if ($TenantFilter -and $TenantFilter -ne 'AllTenants') {
29+
$Conditions.Add("RowKey gt '$($TenantFilter)' and RowKey lt '$($TenantFilter)~'")
2230
}
2331

2432
$Filter = $Conditions -join ' and '
@@ -27,13 +35,60 @@ function Get-CIPPBackup {
2735

2836
if ($NameOnly.IsPresent) {
2937
$Info = $Info | Where-Object { $_.RowKey -notmatch '-part[0-9]+$' }
30-
if ($TenantFilter) {
31-
$Info = $Info | Where-Object { $_.RowKey -match "^$($TenantFilter)_" }
32-
}
3338
} else {
3439
if ($TenantFilter -and $TenantFilter -ne 'AllTenants') {
3540
$Info = $Info | Where-Object { $_.TenantFilter -eq $TenantFilter }
3641
}
3742
}
43+
44+
# Augment results with blob-link awareness and fetch blob content when needed
45+
if (-not $NameOnly.IsPresent -and $Info) {
46+
foreach ($item in $Info) {
47+
$isBlobLink = $false
48+
$blobPath = $null
49+
if ($null -ne $item.PSObject.Properties['Backup']) {
50+
$b = $item.Backup
51+
if ($b -is [string] -and ($b -like 'https://*' -or $b -like 'http://*')) {
52+
$isBlobLink = $true
53+
$blobPath = $b
54+
55+
# Fetch the actual backup content from blob storage
56+
try {
57+
# Extract container/blob path from URL
58+
$resourcePath = $blobPath
59+
if ($resourcePath -like '*:10000/*') {
60+
# Azurite format: http://host:10000/devstoreaccount1/container/blob
61+
$parts = $resourcePath -split ':10000/'
62+
if ($parts.Count -gt 1) {
63+
# Remove account name to get container/blob
64+
$resourcePath = ($parts[1] -split '/', 2)[-1]
65+
}
66+
} elseif ($resourcePath -like '*blob.core.windows.net/*') {
67+
# Azure Storage format: https://account.blob.core.windows.net/container/blob
68+
$resourcePath = ($resourcePath -split '.blob.core.windows.net/', 2)[-1]
69+
}
70+
71+
# Download the blob content
72+
$ConnectionString = $env:AzureWebJobsStorage
73+
$blobResponse = New-CIPPAzStorageRequest -Service 'blob' -Resource $resourcePath -Method 'GET' -ConnectionString $ConnectionString
74+
75+
if ($blobResponse -and $blobResponse.Bytes) {
76+
$backupContent = [System.Text.Encoding]::UTF8.GetString($blobResponse.Bytes)
77+
# Replace the URL with the actual backup content
78+
$item.Backup = $backupContent
79+
Write-Verbose "Successfully retrieved backup content from blob storage for $($item.RowKey)"
80+
} else {
81+
Write-Warning "Failed to retrieve backup content from blob storage for $($item.RowKey)"
82+
}
83+
} catch {
84+
Write-Warning "Error fetching backup from blob storage: $($_.Exception.Message)"
85+
# Leave the URL in place if we can't fetch the content
86+
}
87+
}
88+
}
89+
$item | Add-Member -NotePropertyName 'BackupIsBlobLink' -NotePropertyValue $isBlobLink -Force
90+
$item | Add-Member -NotePropertyName 'BlobResourcePath' -NotePropertyValue $blobPath -Force
91+
}
92+
}
3893
return $Info
3994
}

0 commit comments

Comments
 (0)