Merge branch 'main' into v16/feature/sign-providers

# Conflicts:
#	src/Umbraco.Cms.Api.Management/Controllers/DataType/Tree/SiblingsDataTypeTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/Document/Tree/SiblingsDocumentTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/DocumentBlueprint/Tree/SiblingsDocumentBlueprintTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/DocumentType/Tree/SiblingsDocumentTypeTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/Media/Tree/SiblingsMediaTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/MediaType/Tree/SiblingsMediaTypeTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/Template/Tree/SiblingsTemplateTreeController.cs
#	src/Umbraco.Cms.Api.Management/Controllers/Tree/EntityTreeControllerBase.cs
#	src/Umbraco.Cms.Api.Management/Controllers/Tree/UserStartNodeTreeControllerBase.cs

Author: AndyButland

build/azure-pipelines.yml

@@ -513,9 +513,9 @@ stages:
       UMBRACO__CMS__WEBROUTING__UMBRACOAPPLICATIONURL: https://localhost:44331/
       ASPNETCORE_URLS: https://localhost:44331
     jobs:
-      # E2E Tests
+      # E2E Smoke Tests
       - job:
-        displayName: E2E Tests (SQLite)
+        displayName: E2E Smoke Tests (SQLite)
         # currently disabled due to DB locks randomly occuring.
         condition: eq(${{parameters.sqliteAcceptanceTests}}, True)
         variables:
@@ -678,7 +678,7 @@ stages:
               testRunTitle: "$(Agent.JobName)"
 
       - job:
-        displayName: E2E Tests (SQL Server)
+        displayName: E2E Smoke Tests (SQL Server)
         variables:
           # Connection string
           CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
@@ -862,6 +862,176 @@ stages:
               searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
               testRunTitle: "$(Agent.JobName)"
 
+      - job:
+        displayName: E2E Release Tests (SQL Server)
+        variables:
+          # Connection string
+          CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
+          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
+        condition: eq(dependencies.Build.outputs['A.build.NBGV_PublicRelease'], 'True')
+        strategy:
+          matrix:
+            WindowsPart1Of3:
+              vmImage: "windows-latest"
+              testCommand: "npm run releaseTest -- --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npm run releaseTest -- --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npm run releaseTest -- --shard=3/3"
+        pool:
+          vmImage: $(vmImage)
+        steps:
+          # Setup test environment
+          - task: DownloadPipelineArtifact@2
+            displayName: Download NuGet artifacts
+            inputs:
+              artifact: nupkg
+              path: $(Agent.BuildDirectory)/app/nupkg
+
+          - task: NodeTool@0
+            displayName: Use Node.js $(nodeVersion)
+            inputs:
+              versionSpec: $(nodeVersion)
+
+          - task: UseDotNet@2
+            displayName: Use .NET SDK from global.json
+            inputs:
+              useGlobalJson: true
+
+          - pwsh: |
+              "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL)
+              UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+              URL=$(ASPNETCORE_URLS)
+              STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json
+              CONSOLE_ERRORS_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/console-errors.json" | Out-File .env
+            displayName: Generate .env
+            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
+
+          # Cache and restore NPM packages
+          - task: Cache@2
+            displayName: Cache NPM packages
+            inputs:
+              key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
+              restoreKeys: |
+                npm_e2e | "$(Agent.OS)"
+                npm_e2e
+              path: $(npm_config_cache)
+
+          - script: npm ci --no-fund --no-audit --prefer-offline
+            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
+            displayName: Restore NPM packages
+
+          # Build application
+          - pwsh: |
+              $cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g"
+              dotnet new nugetconfig
+              dotnet nuget add source ./nupkg --name Local
+              dotnet new install Umbraco.Templates::$cmsVersion
+              dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check
+              dotnet restore UmbracoProject
+              cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject
+              dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore
+              dotnet dev-certs https
+            displayName: Build application
+            workingDirectory: $(Agent.BuildDirectory)/app
+
+          # Start SQL Server
+          - powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
+            displayName: Start SQL Server Docker image (Linux)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+
+          - pwsh: SqlLocalDB start MSSQLLocalDB
+            displayName: Start SQL Server LocalDB (Windows)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+
+          # Run application
+          - bash: |
+              nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 &
+              echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!"
+            displayName: Run application (Linux)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+            workingDirectory: $(Agent.BuildDirectory)/app
+
+          - pwsh: |
+              $process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
+              Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
+            displayName: Run application (Windows)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+            workingDirectory: $(Agent.BuildDirectory)/app
+
+          # Wait for application to start responding to requests
+          - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
+            displayName: Wait for application
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+
+          # Install Playwright and dependencies
+          - pwsh: npx playwright install chromium
+            displayName: Install Playwright only with Chromium browser
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+
+          # Test
+          - pwsh: $(testCommand)
+            displayName: Run Playwright tests
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+            env:
+              CI: true
+              CommitId: $(Build.SourceVersion)
+              AgentOs: $(Agent.OS)
+
+          # Stop application
+          - bash: kill -15 $(AcceptanceTestProcessId)
+            displayName: Stop application (Linux)
+            condition: and(ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux'))
+
+          - pwsh: Stop-Process -Id $(AcceptanceTestProcessId)
+            displayName: Stop application (Windows)
+            condition: and(ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT'))
+
+          # Stop SQL Server
+          - pwsh: docker stop mssql
+            displayName: Stop SQL Server Docker image (Linux)
+            condition: eq(variables['Agent.OS'], 'Linux')
+
+          - pwsh: SqlLocalDB stop MSSQLLocalDB
+            displayName: Stop SQL Server LocalDB (Windows)
+            condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+          # Copy artifacts
+          - pwsh: |
+              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
+              }
+            displayName: Copy Playwright results
+            condition: succeededOrFailed()
+
+          # Copy console error log
+          - pwsh: |
+              if (Test-Path tests/Umbraco.Tests.AcceptanceTest/console-errors.json) {
+                  Copy-Item tests/Umbraco.Tests.AcceptanceTest/console-errors.json $(Build.ArtifactStagingDirectory)
+              }
+            displayName: Copy console error log
+            condition: succeededOrFailed()
+
+          # Publish test artifacts
+          - task: PublishPipelineArtifact@1
+            displayName: Publish test artifacts
+            condition: succeededOrFailed()
+            inputs:
+              targetPath: $(Build.ArtifactStagingDirectory)
+              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
+
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: "Publish test results"
+            condition: succeededOrFailed()
+            inputs:
+              testResultsFormat: 'JUnit'
+              testResultsFiles: '*.xml'
+              searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
+              testRunTitle: "$(Agent.JobName)"
+
   ###############################################
   ## Release
   ###############################################
@@ -1067,4 +1237,4 @@ stages:
               storage: umbracoapidocs
               ContainerName: "$web"
               BlobPrefix: v$(umbracoMajorVersion)/ui-api
-              CleanTargetBeforeCopy: true
+              CleanTargetBeforeCopy: true

build/nightly-E2E-test-pipelines.yml

@@ -9,11 +9,10 @@ schedules:
     branches:
       include:
         - v15/dev
-        - release/16.0
         - main
 
 parameters:
-    # Skipped due to DB locks
+  # Skipped due to DB locks
   - name: sqliteAcceptanceTests
     displayName: Run SQLite Acceptance Tests
     type: boolean
@@ -484,3 +483,51 @@ stages:
               testResultsFiles: '*.xml'
               searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
               testRunTitle: "$(Agent.JobName)"
+
+  - stage: NotifySlackBot
+    displayName: Notify Slack on Failure
+    dependsOn: E2E
+    # This stage will only run if the E2E tests fail or succeed with issues
+    condition: or(
+      eq(dependencies.E2E.result, 'failed'),
+      eq(dependencies.E2E.result, 'succeededWithIssues'))
+    jobs:
+      - job: PostToSlack
+        displayName: Send Slack Notification
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          # We send a payload to the Slack webhook URL, which will post a message to a specific channel
+          - bash: |
+              PROJECT_NAME_ENCODED=$(echo -n "$SYSTEM_TEAMPROJECT" | jq -s -R -r @uri)
+              PIPELINE_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${PROJECT_NAME_ENCODED}/_build/results?buildId=${BUILD_BUILDID}&view=ms.vss-test-web.build-test-results-tab"
+
+              PAYLOAD="{
+                \"attachments\": [
+                  {
+                    \"color\": \"#ff0000\",
+                    \"pretext\": \"Nightly E2E pipeline *${BUILD_DEFINITIONNAME}* (#${BUILD_BUILDNUMBER}) failed!\",
+                    \"title\": \"View Failed E2E Test Results\",
+                    \"title_link\": \"$PIPELINE_URL\",
+                    \"fields\": [
+                      {
+                        \"title\": \"Pipeline\",
+                        \"value\": \"${BUILD_DEFINITIONNAME}\",
+                        \"short\": true
+                      },
+                      {
+                        \"title\": \"Build ID\",
+                        \"value\": \"${BUILD_BUILDID}\",
+                        \"short\": true
+                      }
+                    ]
+                  }
+                ]
+              }"
+
+              echo "Sending Slack message to: $PIPELINE_URL"
+              curl -X POST -H 'Content-type: application/json' \
+                --data "$PAYLOAD" \
+                "$SLACK_WEBHOOK_URL"
+            env:
+              SLACK_WEBHOOK_URL: $(E2ESLACKWEBHOOKURL)

src/Umbraco.Cms.Api.Common/ViewModels/Pagination/SubsetViewModel.cs

@@ -0,0 +1,17 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Umbraco.Cms.Api.Common.ViewModels.Pagination;
+
+public class SubsetViewModel<T>
+{
+    [Required]
+    public long TotalBefore { get; set; }
+
+    [Required]
+    public long TotalAfter { get; set; }
+
+    [Required]
+    public IEnumerable<T> Items { get; set; } = Enumerable.Empty<T>();
+
+    public static SubsetViewModel<T> Empty() => new();
+}

src/Umbraco.Cms.Api.Delivery/Caching/DeliveryApiOutputCachePolicy.cs

@@ -23,7 +23,12 @@ ValueTask IOutputCachePolicy.CacheRequestAsync(OutputCacheContext context, Cance
             .RequestServices
             .GetRequiredService<IRequestPreviewService>();
 
-        context.EnableOutputCaching = requestPreviewService.IsPreview() is false;
+        IApiAccessService apiAccessService = context
+            .HttpContext
+            .RequestServices
+            .GetRequiredService<IApiAccessService>();
+
+        context.EnableOutputCaching = requestPreviewService.IsPreview() is false && apiAccessService.HasPublicAccess();
         context.ResponseExpirationTimeSpan = _duration;
         context.CacheVaryByRules.HeaderNames = _varyByHeaderNames;
 

src/Umbraco.Cms.Api.Management/Controllers/DataType/Tree/SiblingsDataTypeTreeController.cs

@@ -1,6 +1,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.DependencyInjection;
+using Umbraco.Cms.Api.Common.ViewModels.Pagination;
 using Umbraco.Cms.Api.Management.Services.Signs;
 using Umbraco.Cms.Api.Management.ViewModels.Tree;
 using Umbraco.Cms.Core.Services;
@@ -22,7 +23,10 @@ public SiblingsDataTypeTreeController(IEntityService entityService, SignProvider
     }
 
     [HttpGet("siblings")]
-    [ProducesResponseType(typeof(IEnumerable<DataTypeTreeItemResponseModel>), StatusCodes.Status200OK)]
-    public Task<ActionResult<IEnumerable<DataTypeTreeItemResponseModel>>> Siblings(CancellationToken cancellationToken, Guid target, int before, int after)
-        => GetSiblings(target, before, after);
+    [ProducesResponseType(typeof(SubsetViewModel<DataTypeTreeItemResponseModel>), StatusCodes.Status200OK)]
+    public async Task<ActionResult<SubsetViewModel<DataTypeTreeItemResponseModel>>> Siblings(CancellationToken cancellationToken, Guid target, int before, int after, bool foldersOnly = false)
+    {
+        RenderFoldersOnly(foldersOnly);
+        return await GetSiblings(target, before, after);
+    }
 }

src/Umbraco.Cms.Api.Management/Controllers/Document/Tree/SiblingsDocumentTreeController.cs

@@ -2,6 +2,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.DependencyInjection;
+using Umbraco.Cms.Api.Common.ViewModels.Pagination;
 using Umbraco.Cms.Api.Management.Factories;
 using Umbraco.Cms.Api.Management.Services.Entities;
 using Umbraco.Cms.Api.Management.Services.Signs;
@@ -59,7 +60,10 @@ public SiblingsDocumentTreeController(
 
     [HttpGet("siblings")]
     [MapToApiVersion("1.0")]
-    [ProducesResponseType(typeof(IEnumerable<DocumentTreeItemResponseModel>), StatusCodes.Status200OK)]
-    public Task<ActionResult<IEnumerable<DocumentTreeItemResponseModel>>> Siblings(CancellationToken cancellationToken, Guid target, int before, int after)
-        => GetSiblings(target, before, after);
+    [ProducesResponseType(typeof(SubsetViewModel<DocumentTreeItemResponseModel>), StatusCodes.Status200OK)]
+    public async Task<ActionResult<SubsetViewModel<DocumentTreeItemResponseModel>>> Siblings(CancellationToken cancellationToken, Guid target, int before, int after, Guid? dataTypeId = null)
+    {
+        IgnoreUserStartNodesForDataType(dataTypeId);
+        return await GetSiblings(target, before, after);
+    }
 }

src/Umbraco.Cms.Api.Management/Controllers/DocumentBlueprint/Tree/SiblingsDocumentBlueprintTreeController.cs

@@ -1,6 +1,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.DependencyInjection;
+using Umbraco.Cms.Api.Common.ViewModels.Pagination;
 using Umbraco.Cms.Api.Management.Factories;
 using Umbraco.Cms.Api.Management.Services.Signs;
 using Umbraco.Cms.Api.Management.ViewModels.Tree;
@@ -23,11 +24,15 @@ public SiblingsDocumentBlueprintTreeController(IEntityService entityService, Sig
     }
 
     [HttpGet("siblings")]
-    [ProducesResponseType(typeof(IEnumerable<DocumentBlueprintTreeItemResponseModel>), StatusCodes.Status200OK)]
-    public Task<ActionResult<IEnumerable<DocumentBlueprintTreeItemResponseModel>>> Siblings(
+    [ProducesResponseType(typeof(SubsetViewModel<DocumentBlueprintTreeItemResponseModel>), StatusCodes.Status200OK)]
+    public async Task<ActionResult<SubsetViewModel<DocumentBlueprintTreeItemResponseModel>>> Siblings(
         CancellationToken cancellationToken,
         Guid target,
         int before,
-        int after) =>
-        GetSiblings(target, before, after);
+        int after,
+        bool foldersOnly = false)
+    {
+        RenderFoldersOnly(foldersOnly);
+        return await GetSiblings(target, before, after);
+    }
 }