V15: Add custom serializer for hybrid cache (#17727)

* Add custom serializer

* Add migration to rebuild cache

* Rename migration namespace to 15.1

* Also clear media cache

* Remove failed cache items

* Refactor to only use keys for document cache repository

---------

Co-authored-by: nikolajlauridsen <[email protected]>

Author: Zeegaan

src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs

@@ -104,5 +104,6 @@ protected virtual void DefinePlan()
         To<V_15_0_0.ConvertBlockGridEditorProperties>("{9D3CE7D4-4884-41D4-98E8-302EB6CB0CF6}");
         To<V_15_0_0.ConvertRichTextEditorProperties>("{37875E80-5CDD-42FF-A21A-7D4E3E23E0ED}");
         To<V_15_0_0.ConvertLocalLinks>("{42E44F9E-7262-4269-922D-7310CB48E724}");
+        To<V_15_1_0.RebuildCacheMigration>("{7B51B4DE-5574-4484-993E-05D12D9ED703}");
     }
 }

src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/RebuildCacheMigration.cs

@@ -0,0 +1,23 @@
+using Umbraco.Cms.Core.PublishedCache;
+
+namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_1_0;
+
+[Obsolete("Will be removed in V18")]
+public class RebuildCacheMigration : MigrationBase
+{
+    private readonly IDocumentCacheService _documentCacheService;
+    private readonly IMediaCacheService _mediaCacheService;
+
+    public RebuildCacheMigration(IMigrationContext context, IDocumentCacheService documentCacheService, IMediaCacheService mediaCacheService) : base(context)
+    {
+        _documentCacheService = documentCacheService;
+        _mediaCacheService = mediaCacheService;
+    }
+
+    protected override void Migrate()
+    {
+        _documentCacheService.ClearMemoryCacheAsync(CancellationToken.None).GetAwaiter().GetResult();
+        _mediaCacheService.ClearMemoryCacheAsync(CancellationToken.None).GetAwaiter().GetResult();
+    }
+
+}

src/Umbraco.PublishedCache.HybridCache/ContentCacheNode.cs

@@ -4,7 +4,7 @@ namespace Umbraco.Cms.Infrastructure.HybridCache;
 
 // This is for cache performance reasons, see https://learn.microsoft.com/en-us/aspnet/core/performance/caching/hybrid?view=aspnetcore-9.0#reuse-objects
 [ImmutableObject(true)]
-internal sealed class ContentCacheNode
+public sealed class ContentCacheNode
 {
     public int Id { get; set; }
 

src/Umbraco.PublishedCache.HybridCache/ContentData.cs

@@ -7,7 +7,7 @@ namespace Umbraco.Cms.Infrastructure.HybridCache;
 /// </summary>
 // This is for cache performance reasons, see https://learn.microsoft.com/en-us/aspnet/core/performance/caching/hybrid?view=aspnetcore-9.0#reuse-objects
 [ImmutableObject(true)]
-internal sealed class ContentData
+public sealed class ContentData
 {
     public ContentData(string? name, string? urlSegment, int versionId, DateTime versionDate, int writerId, int? templateId, bool published, Dictionary<string, PropertyData[]>? properties, IReadOnlyDictionary<string, CultureVariation>? cultureInfos)
     {

src/Umbraco.PublishedCache.HybridCache/DependencyInjection/UmbracoBuilderExtensions.cs

@@ -1,7 +1,6 @@
 
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
-using Umbraco.Cms.Core.Cache;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Notifications;
@@ -35,7 +34,7 @@ public static IUmbracoBuilder AddUmbracoHybridCache(this IUmbracoBuilder builder
             // We'll be a bit friendlier and default this to a higher value, you quickly hit the 1MB limit with a few languages and especially blocks.
             // This can be overwritten later if needed.
             options.MaximumPayloadBytes = 1024 * 1024 * 100; // 100MB
-        });
+        }).AddSerializer<ContentCacheNode, HybridCacheSerializer>();
 #pragma warning restore EXTEXP0018
         builder.Services.AddSingleton<IDatabaseCacheRepository, DatabaseCacheRepository>();
         builder.Services.AddSingleton<IPublishedContentCache, DocumentCache>();

src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs

@@ -189,30 +189,6 @@ AND cmsContentNu.nodeId IS NULL
         return count == 0;
     }
 
-    public async Task<ContentCacheNode?> GetContentSourceAsync(int id, bool preview = false)
-    {
-        Sql<ISqlContext>? sql = SqlContentSourcesSelect()
-            .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document))
-            .Append(SqlWhereNodeId(SqlContext, id))
-            .Append(SqlOrderByLevelIdSortOrder(SqlContext));
-
-        ContentSourceDto? dto = await Database.FirstOrDefaultAsync<ContentSourceDto>(sql);
-
-        if (dto == null)
-        {
-            return null;
-        }
-
-        if (preview is false && dto.PubDataRaw is null && dto.PubData is null)
-        {
-            return null;
-        }
-
-        IContentCacheDataSerializer serializer =
-            _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
-        return CreateContentNodeKit(dto, serializer, preview);
-    }
-
     public async Task<ContentCacheNode?> GetContentSourceAsync(Guid key, bool preview = false)
     {
         Sql<ISqlContext>? sql = SqlContentSourcesSelect()
@@ -292,25 +268,6 @@ public IEnumerable<ContentCacheNode> GetContentByContentTypeKey(IEnumerable<Guid
     public IEnumerable<Guid> GetDocumentKeysByContentTypeKeys(IEnumerable<Guid> keys, bool published = false)
         => GetContentSourceByDocumentTypeKey(keys, Constants.ObjectTypes.Document).Where(x => x.Published == published).Select(x => x.Key);
 
-    public async Task<ContentCacheNode?> GetMediaSourceAsync(int id)
-    {
-        Sql<ISqlContext>? sql = SqlMediaSourcesSelect()
-            .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media))
-            .Append(SqlWhereNodeId(SqlContext, id))
-            .Append(SqlOrderByLevelIdSortOrder(SqlContext));
-
-        ContentSourceDto? dto = await Database.FirstOrDefaultAsync<ContentSourceDto>(sql);
-
-        if (dto is null)
-        {
-            return null;
-        }
-
-        IContentCacheDataSerializer serializer =
-            _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
-        return CreateMediaNodeKit(dto, serializer);
-    }
-
     public async Task<ContentCacheNode?> GetMediaSourceAsync(Guid key)
     {
         Sql<ISqlContext>? sql = SqlMediaSourcesSelect()

src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs

@@ -7,12 +7,8 @@ internal interface IDatabaseCacheRepository
 {
     Task DeleteContentItemAsync(int id);
 
-    Task<ContentCacheNode?> GetContentSourceAsync(int id, bool preview = false);
-
     Task<ContentCacheNode?> GetContentSourceAsync(Guid key, bool preview = false);
 
-    Task<ContentCacheNode?> GetMediaSourceAsync(int id);
-
     Task<ContentCacheNode?> GetMediaSourceAsync(Guid key);
 
 

src/Umbraco.PublishedCache.HybridCache/Serialization/HybridCacheSerializer.cs

@@ -0,0 +1,40 @@
+using System.Buffers;
+using MessagePack;
+using MessagePack.Resolvers;
+using Microsoft.Extensions.Caching.Hybrid;
+using Microsoft.Extensions.Logging;
+
+namespace Umbraco.Cms.Infrastructure.HybridCache.Serialization;
+
+internal class HybridCacheSerializer : IHybridCacheSerializer<ContentCacheNode>
+{
+    private readonly ILogger<HybridCacheSerializer> _logger;
+    private readonly MessagePackSerializerOptions _options;
+
+    public HybridCacheSerializer(ILogger<HybridCacheSerializer> logger)
+    {
+        _logger = logger;
+        MessagePackSerializerOptions defaultOptions = ContractlessStandardResolver.Options;
+        IFormatterResolver resolver = CompositeResolver.Create(defaultOptions.Resolver);
+
+        _options = defaultOptions
+            .WithResolver(resolver)
+            .WithCompression(MessagePackCompression.Lz4BlockArray)
+            .WithSecurity(MessagePackSecurity.UntrustedData);
+    }
+
+    public ContentCacheNode Deserialize(ReadOnlySequence<byte> source)
+    {
+        try
+        {
+            return MessagePackSerializer.Deserialize<ContentCacheNode>(source, _options);
+        }
+        catch (MessagePackSerializationException ex)
+        {
+            _logger.LogError(ex, "Error deserializing ContentCacheNode");
+            return null!;
+        }
+    }
+
+    public void Serialize(ContentCacheNode value, IBufferWriter<byte> target) => target.Write(MessagePackSerializer.Serialize(value, _options));
+}

src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs

@@ -74,23 +74,7 @@ public DocumentCacheService(
     {
         bool calculatedPreview = preview ?? GetPreview();
 
-        ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync(
-            GetCacheKey(key, calculatedPreview), // Unique key to the cache entry
-            async cancel =>
-            {
-                using ICoreScope scope = _scopeProvider.CreateCoreScope();
-                ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, calculatedPreview);
-                scope.Complete();
-                return contentCacheNode;
-            },
-            GetEntryOptions(key));
-
-        return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedContent(contentCacheNode, calculatedPreview).CreateModel(_publishedModelFactory);
-    }
-
-    private bool GetPreview()
-    {
-        return _previewService.IsInPreview();
+        return await GetNodeAsync(key, calculatedPreview);
     }
 
     public async Task<IPublishedContent?> GetByIdAsync(int id, bool? preview = null)
@@ -104,17 +88,37 @@ private bool GetPreview()
         bool calculatedPreview = preview ?? GetPreview();
         Guid key = keyAttempt.Result;
 
+        return await GetNodeAsync(key, calculatedPreview);
+    }
+
+    private async Task<IPublishedContent?> GetNodeAsync(Guid key, bool preview)
+    {
+        var cacheKey = GetCacheKey(key, preview);
+
         ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync(
-            GetCacheKey(keyAttempt.Result, calculatedPreview), // Unique key to the cache entry
-        async cancel =>
+            cacheKey,
+            async cancel =>
+            {
+                using ICoreScope scope = _scopeProvider.CreateCoreScope();
+                ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, preview);
+                scope.Complete();
+                return contentCacheNode;
+            },
+            GetEntryOptions(key));
+
+        // We don't want to cache removed items, this may cause issues if the L2 serializer changes.
+        if (contentCacheNode is null)
         {
-            using ICoreScope scope = _scopeProvider.CreateCoreScope();
-            ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(id, calculatedPreview);
-            scope.Complete();
-            return contentCacheNode;
-        }, GetEntryOptions(key));
+            await _hybridCache.RemoveAsync(cacheKey);
+            return null;
+        }
+
+        return _publishedContentFactory.ToIPublishedContent(contentCacheNode, preview).CreateModel(_publishedModelFactory);
+    }
 
-        return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedContent(contentCacheNode, calculatedPreview).CreateModel(_publishedModelFactory);;
+    private bool GetPreview()
+    {
+        return _previewService.IsInPreview();
     }
 
     public IEnumerable<IPublishedContent> GetByContentType(IPublishedContentType contentType)

src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs

@@ -76,17 +76,7 @@ public MediaCacheService(
             return null;
         }
 
-        ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync(
-            $"{key}", // Unique key to the cache entry
-            async cancel =>
-            {
-                using ICoreScope scope = _scopeProvider.CreateCoreScope();
-                ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(idAttempt.Result);
-                scope.Complete();
-                return mediaCacheNode;
-            }, GetEntryOptions(key));
-
-        return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory);
+        return await GetNodeAsync(key);
     }
 
     public async Task<IPublishedContent?> GetByIdAsync(int id)
@@ -96,19 +86,33 @@ public MediaCacheService(
         {
             return null;
         }
+
         Guid key = keyAttempt.Result;
 
+        return await GetNodeAsync(key);
+    }
+
+    private async Task<IPublishedContent?> GetNodeAsync(Guid key)
+    {
+        var cacheKey = $"{key}";
         ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync(
-            $"{keyAttempt.Result}", // Unique key to the cache entry
+            cacheKey, // Unique key to the cache entry
             async cancel =>
             {
                 using ICoreScope scope = _scopeProvider.CreateCoreScope();
-                ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(id);
+                ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(key);
                 scope.Complete();
                 return mediaCacheNode;
             }, GetEntryOptions(key));
 
-        return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory);
+        // We don't want to cache removed items, this may cause issues if the L2 serializer changes.
+        if (contentCacheNode is null)
+        {
+            await _hybridCache.RemoveAsync(cacheKey);
+            return null;
+        }
+
+        return _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory);
     }
 
     public async Task<bool> HasContentByIdAsync(int id)