v9: Move local xml package files to database instead (#11654)

* Move package to database

* Added migration and implemented new repository

* Updated migrations to use proper xml convert

* Fixed save function and renamed createDate to update date

* Updated dependencyInjection

* Updated UmbracoPlan.cs

* Apply suggestions from code review

Co-authored-by: Elitsa Marinovska <[email protected]>

* Added File check

* Tried using same context as create table

* Fix DTO

* Fix GetById and local package saving

* Fix images when migrating

* Implement deletion of all local files

* Only delete local repo file, not file snapshots

* Remove static package path and use the one we save

* Update package repo to export package and remove check for ids when exporting

* Minor fixes

* Fix so that you can download package after creating

* Update savePackage method to export package afterwards

Co-authored-by: Nikolaj Geisle <[email protected]>
Co-authored-by: Elitsa Marinovska <[email protected]>
Co-authored-by: Elitsa Marinovska <[email protected]>

Author: 3 people

src/Umbraco.Core/Extensions/ContentExtensions.cs

@@ -367,7 +367,7 @@ public static IEnumerable<string> GetDirtyUserProperties(this IContentBase entit
         /// <param name="content"><see cref="IContent"/> to generate xml for</param>
         /// <param name="serializer"></param>
         /// <returns>Xml representation of the passed in <see cref="IContent"/></returns>
-        internal static XElement ToDeepXml(this IContent content, IEntityXmlSerializer serializer)
+        public static XElement ToDeepXml(this IContent content, IEntityXmlSerializer serializer)
         {
             return serializer.Serialize(content, false, true);
         }

src/Umbraco.Core/Packaging/PackagesRepository.cs

@@ -21,6 +21,7 @@ namespace Umbraco.Cms.Core.Packaging
     /// <summary>
     /// Manages the storage of installed/created package definitions
     /// </summary>
+    [Obsolete("Packages have now been moved to the database instead of local files, please use CreatedPackageSchemaRepository instead")]
     public class PackagesRepository : ICreatedPackagesRepository
     {
         private readonly IContentService _contentService;
@@ -744,5 +745,13 @@ private XDocument EnsureStorage(out string packagesFile)
             var packagesXml = XDocument.Load(packagesFile);
             return packagesXml;
         }
+
+        public void DeleteLocalRepositoryFiles()
+        {
+            var packagesFile = _hostingEnvironment.MapPathContentRoot(CreatedPackagesFile);
+            File.Delete(packagesFile);
+            var packagesFolder = _hostingEnvironment.MapPathContentRoot(_packagesFolderPath);
+            Directory.Delete(packagesFolder);
+        }
     }
 }

src/Umbraco.Core/Persistence/Constants-DatabaseSchema.cs

@@ -81,6 +81,8 @@ public static class Tables
                 public const string UserLogin = TableNamePrefix + "UserLogin";
 
                 public const string LogViewerQuery = TableNamePrefix + "LogViewerQuery";
+
+                public const string CreatedPackageSchema = TableNamePrefix + "CreatedPackageSchema";
             }
         }
     }

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Services.cs

@@ -15,6 +15,8 @@
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Services.Implement;
 using Umbraco.Cms.Infrastructure.Packaging;
+using Umbraco.Cms.Infrastructure.Persistence;
+using Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement;
 using Umbraco.Cms.Infrastructure.Services.Implement;
 using Umbraco.Extensions;
 
@@ -74,16 +76,14 @@ internal static IUmbracoBuilder AddServices(this IUmbracoBuilder builder)
 
             builder.Services.AddUnique<ConflictingPackageData>();
             builder.Services.AddUnique<CompiledPackageXmlParser>();
-            builder.Services.AddUnique<ICreatedPackagesRepository>(factory => CreatePackageRepository(factory, "createdPackages.config"));
+            builder.Services.AddUnique(factory => CreatePackageRepository(factory, "createdPackages.config"));
+            builder.Services.AddUnique<ICreatedPackagesRepository, CreatedPackageSchemaRepository>();
             builder.Services.AddUnique<PackageDataInstallation>();
             builder.Services.AddUnique<IPackageInstallation, PackageInstallation>();
 
             return builder;
         }
 
-        /// <summary>
-        /// Creates an instance of PackagesRepository for either the ICreatedPackagesRepository or the IInstalledPackagesRepository
-        /// </summary>
         private static PackagesRepository CreatePackageRepository(IServiceProvider factory, string packageRepoFileName)
             => new PackagesRepository(
                 factory.GetRequiredService<IContentService>(),

src/Umbraco.Infrastructure/Migrations/Install/DatabaseSchemaCreator.cs

@@ -78,7 +78,8 @@ public class DatabaseSchemaCreator
             typeof(ContentScheduleDto),
             typeof(LogViewerQueryDto),
             typeof(ContentVersionCleanupPolicyDto),
-            typeof(UserGroup2NodeDto)
+            typeof(UserGroup2NodeDto),
+            typeof(CreatedPackageSchemaDto)
         };
 
         private readonly IUmbracoDatabase _database;

src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs

@@ -268,6 +268,7 @@ protected void DefinePlan()
             // TO 9.2.0
             To<AddUserGroup2NodeTable>("{0571C395-8F0B-44E9-8E3F-47BDD08D817B}");
             To<AddDefaultForNotificationsToggle>("{AD3D3B7F-8E74-45A4-85DB-7FFAD57F9243}");
+            To<MovePackageXMLToDb>("{A2F22F17-5870-4179-8A8D-2362AA4A0A5F}");
         }
     }
 }

src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_2_0/MovePackageXMLToDb.cs

@@ -0,0 +1,72 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml;
+using System.Xml.Linq;
+using Umbraco.Cms.Core.Packaging;
+using Umbraco.Cms.Infrastructure.Persistence;
+using Umbraco.Cms.Infrastructure.Persistence.Dtos;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_9_2_0
+{
+    public class MovePackageXMLToDb : MigrationBase
+    {
+        private readonly PackagesRepository _packagesRepository;
+        private readonly PackageDefinitionXmlParser _xmlParser;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MovePackageXMLToDb"/> class.
+        /// </summary>
+        public MovePackageXMLToDb(IMigrationContext context, PackagesRepository packagesRepository)
+            : base(context)
+        {
+            _packagesRepository = packagesRepository;
+            _xmlParser = new PackageDefinitionXmlParser();
+        }
+
+        private void CreateDatabaseTable()
+        {
+            // Add CreatedPackage table in database if it doesn't exist
+            IEnumerable<string> tables = SqlSyntax.GetTablesInSchema(Context.Database);
+            if (!tables.InvariantContains(CreatedPackageSchemaDto.TableName))
+            {
+                Create.Table<CreatedPackageSchemaDto>().Do();
+            }
+        }
+
+        private void MigrateCreatedPackageFilesToDb()
+        {
+            // Load data from file
+            IEnumerable<PackageDefinition> packages = _packagesRepository.GetAll();
+            var createdPackageDtos = new List<CreatedPackageSchemaDto>();
+            foreach (PackageDefinition package in packages)
+            {
+                // Create dto from xmlDocument
+                var dto = new CreatedPackageSchemaDto()
+                {
+                    Name = package.Name,
+                    Value = _xmlParser.ToXml(package).ToString(),
+                    UpdateDate = DateTime.Now,
+                    PackageId = Guid.NewGuid()
+                };
+                createdPackageDtos.Add(dto);
+            }
+
+            _packagesRepository.DeleteLocalRepositoryFiles();
+            if (createdPackageDtos.Any())
+            {
+                // Insert dto into CreatedPackage table
+                Database.InsertBulk(createdPackageDtos);
+            }
+        }
+
+        /// <inheritdoc/>
+        protected override void Migrate()
+        {
+            CreateDatabaseTable();
+            MigrateCreatedPackageFilesToDb();
+        }
+    }
+}

src/Umbraco.Infrastructure/Persistence/Dtos/CreatedPackageSchemaDto.cs

@@ -0,0 +1,38 @@
+using System;
+using NPoco;
+using Umbraco.Cms.Infrastructure.Persistence.DatabaseAnnotations;
+using Umbraco.Cms.Infrastructure.Persistence.DatabaseModelDefinitions;
+
+namespace Umbraco.Cms.Infrastructure.Persistence.Dtos
+{
+    [TableName(TableName)]
+    [ExplicitColumns]
+    [PrimaryKey("id")]
+    public class CreatedPackageSchemaDto
+    {
+        public const string TableName = Cms.Core.Constants.DatabaseSchema.Tables.CreatedPackageSchema;
+
+        [Column("id")]
+        [PrimaryKeyColumn]
+        public int Id { get; set; }
+
+        [Column("name")]
+        [Length(255)]
+        [NullSetting(NullSetting = NullSettings.NotNull)]
+        [Index(IndexTypes.UniqueNonClustered, ForColumns = "name", Name = "IX_" + TableName + "_Name")]
+        public string Name { get; set; }
+
+        [Column("value")]
+        [SpecialDbType(SpecialDbTypes.NVARCHARMAX)]
+        [NullSetting(NullSetting = NullSettings.NotNull)]
+        public string Value { get; set; }
+
+        [Column("updateDate")]
+        [Constraint(Default = SystemMethods.CurrentDateTime)]
+        public DateTime UpdateDate { get; set; }
+
+        [Column("packageId")]
+        [NullSetting(NullSetting = NullSettings.NotNull)]
+        public Guid PackageId { get; set; }
+    }
+}