Merge pull request #1785 from KelvinTegelaar/dev

Dev

Author: KelvinTegelaar

.github/workflows/publish_release.yml

@@ -48,11 +48,36 @@ jobs:
             echo "tag_exists=false" >> $GITHUB_ENV
           fi
 
+      # Get Previous Tag
+      - name: Get Previous Tag
+        id: previous_tag
+        if: env.tag_exists == 'false'
+        run: |
+          PREV_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
+          echo "previous_tag=$PREV_TAG" >> $GITHUB_OUTPUT
+          echo "Previous tag: $PREV_TAG"
+
       # Generate Release Notes
       - name: Generate Release Notes
         id: changelog
         if: env.tag_exists == 'false'
-        uses: mikepenz/[email protected]
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const params = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              tag_name: '${{ steps.get_version.outputs.version }}',
+              target_commitish: context.sha
+            };
+
+            const previousTag = '${{ steps.previous_tag.outputs.previous_tag }}';
+            if (previousTag) {
+              params.previous_tag_name = previousTag;
+            }
+
+            const { data } = await github.rest.repos.generateReleaseNotes(params);
+            core.setOutput('changelog', data.body);
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -75,10 +100,6 @@ jobs:
         if: env.tag_exists == 'false'
         run: |
           mkdir -p src/releases
-          zip -r src/releases/release_${{ steps.get_version.outputs.version }}.zip . \
-            --exclude "./src/releases/*" \
-            --exclude ".*" \
-            --exclude ".*/**"
           zip -r src/releases/latest.zip . \
             --exclude "./src/releases/*" \
             --exclude ".*" \

Modules/CIPPCore/Public/Add-CIPPDbItem.ps1

@@ -69,23 +69,55 @@ function Add-CIPPDbItem {
             if ($ExistingEntities) {
                 Remove-AzDataTableEntity @Table -Entity $ExistingEntities -Force | Out-Null
             }
-            $Entities = foreach ($Item in $Data) {
-                $ItemId = $Item.id ?? $Item.ExternalDirectoryObjectId ?? $Item.Identity ?? $Item.skuId
-                @{
-                    PartitionKey = $TenantFilter
-                    RowKey       = Format-RowKey "$Type-$ItemId"
-                    Data         = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
-                    Type         = $Type
+
+            # Calculate batch size based on available memory
+            $AvailableMemory = [System.GC]::GetTotalMemory($false)
+            $AvailableMemoryMB = [math]::Round($AvailableMemory / 1MB, 2)
+
+            # Estimate item size from first item (with fallback)
+            $EstimatedItemSizeBytes = 1KB # Default assumption
+            if ($Data.Count -gt 0) {
+                $SampleJson = $Data[0] | ConvertTo-Json -Depth 10 -Compress
+                $EstimatedItemSizeBytes = [System.Text.Encoding]::UTF8.GetByteCount($SampleJson)
+            }
+
+            # Use 25% of available memory for batch processing, with min/max bounds
+            $TargetBatchMemoryMB = [Math]::Max(50, $AvailableMemoryMB * 0.25)
+            $CalculatedBatchSize = [Math]::Floor(($TargetBatchMemoryMB * 1MB) / $EstimatedItemSizeBytes)
+            # Reduce max to 500 to prevent OOM with large datasets
+            $BatchSize = [Math]::Max(100, [Math]::Min(500, $CalculatedBatchSize))
+
+            $TotalCount = $Data.Count
+            $ProcessedCount = 0
+            Write-Information "Adding $TotalCount items of type $Type to CIPP Reporting DB for tenant $TenantFilter | Available Memory: ${AvailableMemoryMB}MB | Target Memory: ${TargetBatchMemoryMB}MB | Calculated: $CalculatedBatchSize | Batch Size: $BatchSize (est. item size: $([math]::Round($EstimatedItemSizeBytes/1KB, 2))KB)"
+            for ($i = 0; $i -lt $TotalCount; $i += $BatchSize) {
+                $BatchEnd = [Math]::Min($i + $BatchSize, $TotalCount)
+                $Batch = $Data[$i..($BatchEnd - 1)]
+
+                $Entities = foreach ($Item in $Batch) {
+                    $ItemId = $Item.id ?? $Item.ExternalDirectoryObjectId ?? $Item.Identity ?? $Item.skuId
+                    @{
+                        PartitionKey = $TenantFilter
+                        RowKey       = Format-RowKey "$Type-$ItemId"
+                        Data         = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
+                        Type         = $Type
+                    }
                 }
+
+                Add-CIPPAzDataTableEntity @Table -Entity $Entities -Force | Out-Null
+                $ProcessedCount += $Batch.Count
+
+                # Clear batch variables to free memory
+                $Entities = $null
+                $Batch = $null
+                [System.GC]::Collect()
             }
-            Add-CIPPAzDataTableEntity @Table -Entity $Entities -Force | Out-Null
 
         }
-
         Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Added $($Data.Count) items of type $Type$(if ($Count) { ' (count mode)' })" -sev Debug
 
     } catch {
-        Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Failed to add items of type $Type : $($_.Exception.Message)" -sev Error
+        Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Failed to add items of type $Type : $($_.Exception.Message)" -sev Error -LogData (Get-CippException -Exception $_)
         throw
     }
 }

Modules/CIPPCore/Public/Add-CippTestResult.ps1

@@ -48,7 +48,7 @@ function Add-CippTestResult {
         [string]$TestId,
 
         [Parameter(Mandatory = $false)]
-        [string]$testType = 'Identity',
+        [string]$TestType = 'Identity',
 
         [Parameter(Mandatory = $true)]
         [string]$Status,

Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Mailbox Permissions/Push-StoreMailboxPermissions.ps1

@@ -44,15 +44,21 @@ function Push-StoreMailboxPermissions {
                 # Results are grouped by cmdlet name due to ReturnWithCommand
                 if ($ActualResult['Get-MailboxPermission']) {
                     Write-Information "Adding $($ActualResult['Get-MailboxPermission'].Count) mailbox permissions"
-                    $AllMailboxPermissions.AddRange($ActualResult['Get-MailboxPermission'])
+                    foreach ($perm in $ActualResult['Get-MailboxPermission']) {
+                        $AllMailboxPermissions.Add($perm)
+                    }
                 }
                 if ($ActualResult['Get-RecipientPermission']) {
                     Write-Information "Adding $($ActualResult['Get-RecipientPermission'].Count) recipient permissions"
-                    $AllRecipientPermissions.AddRange($ActualResult['Get-RecipientPermission'])
+                    foreach ($perm in $ActualResult['Get-RecipientPermission']) {
+                        $AllRecipientPermissions.Add($perm)
+                    }
                 }
                 if ($ActualResult['Get-MailboxFolderPermission']) {
                     Write-Information "Adding $($ActualResult['Get-MailboxFolderPermission'].Count) calendar permissions"
-                    $AllCalendarPermissions.AddRange($ActualResult['Get-MailboxFolderPermission'])
+                    foreach ($perm in $ActualResult['Get-MailboxFolderPermission']) {
+                        $AllCalendarPermissions.Add($perm)
+                    }
                 }
             } else {
                 Write-Information "Skipping non-hashtable result: $($ActualResult.GetType().Name)"
@@ -61,30 +67,47 @@ function Push-StoreMailboxPermissions {
 
         # Combine all permissions (mailbox and recipient) into a single collection
         $AllPermissions = [System.Collections.Generic.List[object]]::new()
-        $AllPermissions.AddRange($AllMailboxPermissions)
-        $AllPermissions.AddRange($AllRecipientPermissions)
+        foreach ($perm in $AllMailboxPermissions) {
+            $AllPermissions.Add($perm)
+        }
+        foreach ($perm in $AllRecipientPermissions) {
+            $AllPermissions.Add($perm)
+        }
 
         Write-Information "Aggregated $($AllPermissions.Count) total permissions ($($AllMailboxPermissions.Count) mailbox + $($AllRecipientPermissions.Count) recipient)"
         Write-Information "Aggregated $($AllCalendarPermissions.Count) calendar permissions"
 
         # Store all permissions together as MailboxPermissions
         if ($AllPermissions.Count -gt 0) {
-            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data $AllPermissions
-            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data $AllPermissions -Count
+            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data $AllPermissions.ToArray()
+            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data @{ Count = $AllPermissions.Count } -Count
             Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Cached $($AllPermissions.Count) mailbox permission records" -sev Info
         } else {
             Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'No mailbox permissions found to cache' -sev Info
         }
 
+        # Clear to free memory before processing calendar permissions
+        $AllMailboxPermissions.Clear()
+        $AllRecipientPermissions.Clear()
+        $AllPermissions.Clear()
+        $AllMailboxPermissions = $null
+        $AllRecipientPermissions = $null
+        $AllPermissions = $null
+
         # Store calendar permissions separately
         if ($AllCalendarPermissions.Count -gt 0) {
-            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data $AllCalendarPermissions
-            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data $AllCalendarPermissions -Count
+            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data $AllCalendarPermissions.ToArray()
+            Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data @{ Count = $AllCalendarPermissions.Count } -Count
             Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Cached $($AllCalendarPermissions.Count) calendar permission records" -sev Info
         } else {
             Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'No calendar permissions found to cache' -sev Info
         }
 
+        # Final cleanup
+        $AllCalendarPermissions.Clear()
+        $AllCalendarPermissions = $null
+        [System.GC]::Collect()
+
         return
 
     } catch {

Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-CIPPDBCacheData.ps1

@@ -99,6 +99,7 @@ function Push-CIPPDBCacheData {
                 'ExoPresetSecurityPolicy'
                 'ExoTenantAllowBlockList'
                 'Mailboxes'
+                'CASMailboxes'
                 'MailboxUsage'
                 'OneDriveUsage'
             )
@@ -178,7 +179,6 @@ function Push-CIPPDBCacheData {
             OrchestratorName = "CIPPDBCacheTenant_$TenantFilter"
             Batch            = @($Batch)
             SkipLog          = $true
-            DurableMode      = 'Sequence'
         }
 
         if ($Item.TestRun -eq $true) {

Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Email-Exchange/Reports/Invoke-ListMailboxCAS.ps1

@@ -1,4 +1,4 @@
-Function Invoke-ListMailboxCAS {
+function Invoke-ListMailboxCAS {
     <#
     .FUNCTIONALITY
         Entrypoint
@@ -10,7 +10,7 @@ Function Invoke-ListMailboxCAS {
     # Interact with query parameters or the body of the request.
     $TenantFilter = $Request.Query.TenantFilter
     try {
-        $GraphRequest = New-GraphGetRequest -uri "https://outlook.office365.com/adminapi/beta/$($tenantfilter)/CasMailbox" -Tenantid $tenantfilter -scope ExchangeOnline | Select-Object @{ Name = 'displayName'; Expression = { $_.'DisplayName' } },
+        $GraphRequest = New-ExoRequest -tenantid $TenantFilter -cmdlet 'Get-CasMailbox' | Select-Object @{ Name = 'displayName'; Expression = { $_.'DisplayName' } },
         @{ Name = 'primarySmtpAddress'; Expression = { $_.'PrimarySMTPAddress' } },
         @{ Name = 'ecpenabled'; Expression = { $_.'ECPEnabled' } },
         @{ Name = 'owaenabled'; Expression = { $_.'OWAEnabled' } },

Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Invoke-ListAvailableTests.ps1

@@ -1,7 +1,7 @@
 function Invoke-ListAvailableTests {
     <#
     .FUNCTIONALITY
-        Entrypoint
+        Entrypoint,AnyTenant
     .ROLE
         CIPP.Dashboard.Read
     #>

Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Invoke-ListTestReports.ps1

@@ -4,7 +4,7 @@ function Invoke-ListTestReports {
         Lists all available test reports from JSON files and database
 
     .FUNCTIONALITY
-        Entrypoint
+        Entrypoint,AnyTenant
 
     .ROLE
         Tenant.Reports.Read

Modules/CIPPCore/Public/Get-CippDbRoleMembers.ps1

@@ -10,6 +10,7 @@ function Get-CippDbRoleMembers {
 
     $RoleAssignments = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'RoleAssignmentScheduleInstances'
     $RoleEligibilities = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'RoleEligibilitySchedules'
+    $DirectRoleAssignments = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Roles' | Where-Object { $_.roleTemplateId -eq $RoleTemplateId } | Select-Object -ExpandProperty members
 
     $ActiveMembers = $RoleAssignments | Where-Object {
         $_.roleDefinitionId -eq $RoleTemplateId -and $_.assignmentType -eq 'Assigned'
@@ -45,5 +46,18 @@ function Get-CippDbRoleMembers {
         }
     }
 
+    foreach ($member in $DirectRoleAssignments) {
+        if ($AllMembers.id -notcontains $member.id) {
+            $memberObj = [PSCustomObject]@{
+                id                = $member.id
+                displayName       = $member.displayName
+                userPrincipalName = $member.userPrincipalName
+                '@odata.type'     = $member.'@odata.type'
+                AssignmentType    = 'Direct'
+            }
+            $AllMembers.Add($memberObj)
+        }
+    }
+
     return $AllMembers
 }

Modules/CIPPCore/Public/Set-CIPPDBCacheApps.ps1

@@ -16,6 +16,7 @@ function Set-CIPPDBCacheApps {
         Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Caching applications' -sev Debug
 
         $Apps = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/applications?$top=999&expand=owners' -tenantid $TenantFilter
+        if (!$Apps) { $Apps = @() }
         Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'Apps' -Data $Apps
         Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'Apps' -Data $Apps -Count
         $Apps = $null