Allow persisting entities as documents

Introduce a document-based repository that allows persisting the entire entity payload to a single column.

Regardless of the serialization strategy used, we persist the type full name and assembly version of the persisted entity, which can be invaluable in data migration scenarios.

In addition to the built-in JSON text serialization, we provide three binary serializers too as separate packages: Bson, MessagePack and Protobuf. The last two require annotating the entity as required by the underlying libraries.

Fixes #24.

Author: kzu

.gitignore

@@ -26,4 +26,5 @@ _site
 .jekyll-metadata
 .jekyll-cache
 Gemfile.lock
-package-lock.json
+package-lock.json
+__azurite_db_table__.json

TableStorage.sln

@@ -18,6 +18,20 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "src\Tests\Tests.cs
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Source", "src\TableStorage.Source\TableStorage.Source.csproj", "{B58183AB-38E7-42FA-BE98-5D7B58C06266}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Bson", "src\TableStorage.Bson\TableStorage.Bson.csproj", "{D0C6041C-D796-483A-8470-F78A4C659BD5}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Bson.Source", "src\TableStorage.Bson.Source\TableStorage.Bson.Source.csproj", "{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Source", "Source", "{7412B98E-AA65-4B6E-AD83-6F9960C1C452}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.MessagePack", "src\TableStorage.MessagePack\TableStorage.MessagePack.csproj", "{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.MessagePack.Source", "src\TableStorage.MessagePack.Source\TableStorage.MessagePack.Source.csproj", "{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Protobuf", "src\TableStorage.Protobuf\TableStorage.Protobuf.csproj", "{3DEA170D-5637-4A01-AA53-B727013FA850}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Protobuf.Source", "src\TableStorage.Protobuf.Source\TableStorage.Protobuf.Source.csproj", "{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -36,10 +50,40 @@ Global
 		{B58183AB-38E7-42FA-BE98-5D7B58C06266}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B58183AB-38E7-42FA-BE98-5D7B58C06266}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B58183AB-38E7-42FA-BE98-5D7B58C06266}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D0C6041C-D796-483A-8470-F78A4C659BD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D0C6041C-D796-483A-8470-F78A4C659BD5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D0C6041C-D796-483A-8470-F78A4C659BD5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D0C6041C-D796-483A-8470-F78A4C659BD5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3DEA170D-5637-4A01-AA53-B727013FA850}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3DEA170D-5637-4A01-AA53-B727013FA850}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3DEA170D-5637-4A01-AA53-B727013FA850}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3DEA170D-5637-4A01-AA53-B727013FA850}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{B58183AB-38E7-42FA-BE98-5D7B58C06266} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
+		{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
+		{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
+		{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
+	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9E26BFEF-9184-4EA2-8A64-BCD61E247C33}
 	EndGlobalSection

assets/img/document.png

@@ -119,6 +119,62 @@ This is quite convenient for handling reference data, for example. Enumerating a
 in the partition wouldn't be something you'd typically do for your "real" data, but for 
 reference data, it could be useful.
 
+Stored entities will use individual columns for properties, which makes it easy to browse 
+the data. If you don't need the individual columns, and would like a document-like storage 
+mechanism instead, you can use the `DocumentRepository.Create` and `DocumentPartition.Create` 
+factory methods instead. The API is otherwise the same, but you can see the effect of using 
+one or the other in the following screenshots of the [Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) 
+for the same `Product` entity shown in the first example above:
+
+![Screenshot of entity persisted with separate columns for properties](assets/img/entity.png)
+
+![Screenshot of entity persisted as a document](assets/img/document.png)
+
+The code that persisted both entities is:
+
+```csharp
+var repo = TableRepository.Create<Product>(
+    CloudStorageAccount.DevelopmentStorageAccount,
+    tableName: "Products",
+    partitionKey: p => p.Category,
+    rowKey: p => p.Id);
+
+await repo.PutAsync(new Product("book", "9781473217386")
+{
+    Title = "Neuromancer",
+    Price = 7.32
+});
+
+var docs = DocumentRepository.Create<Product>(
+    CloudStorageAccount.DevelopmentStorageAccount,
+    tableName: "Documents",
+    partitionKey: p => p.Category,
+    rowKey: p => p.Id);
+
+await docs.PutAsync(new Product("book", "9781473217386")
+{
+    Title = "Neuromancer",
+    Price = 7.32
+});
+```
+
+The `DocumentType` is the `Type.FullName` of the entity type, and the `DocumentVersion` is 
+the `Major.Minor` of its assembly, which could be used for advanced data migration scenarios.
+
+In addition to the default built-in JSON plain-text based serializer, you can choose from 
+various binary serializers which will instead persist the document as a byte array:
+
+[![Bson](https://img.shields.io/nuget/v/Devlooped.TableStorage.svg?color=royalblue&label=Bson)](https://www.nuget.org/packages/Devlooped.TableStorage)
+[![MessagePack](https://img.shields.io/nuget/v/Devlooped.TableStorage.svg?color=royalblue&label=MessagePack)](https://www.nuget.org/packages/Devlooped.TableStorage)
+[![Protobuf](https://img.shields.io/nuget/v/Devlooped.TableStorage.svg?color=royalblue&label=Protobuf)](https://www.nuget.org/packages/Devlooped.TableStorage)
+
+You can pass the serializer to use to the factory method as follows: 
+
+```csharp
+var repo = TableRepository.Create<Product>(...,
+    serializer: [BsonDocumentSerializer|MessagePackDocumentSerializer|ProtobufDocumentSerializer].Default);
+```
+
 ### Attributes
 
 If you want to avoid using strings with the factory methods, you can also annotate the 
@@ -128,12 +184,11 @@ entity type to modify the default values used:
 * `[PartitionKey]`: annotates the property that should be used as the partition key
 * `[RowKey]`: annotates the property that should be used as the row key.
 
-Values passed to the `TableRepository.Create<T>` or `TablePartition.Create<T>` override 
-declarative attributes.
+Values passed to the factory methods override declarative attributes.
 
 ### TableEntity Support
 
-Since these repository APIs are quite a bit more intuitive than working against a direct 
+Since these repository APIs are quite a bit more intuitive than working directly against a  
 `TableClient`, you might want to retrieve/enumerate entities just by their built-in `ITableEntity` 
 properties, like `PartitionKey`, `RowKey`, `Timestamp` and `ETag`. For this scenario, we 
 also support creating `ITableRepository<TableEntity>` and `ITablePartition<TableEntity>` 
@@ -161,7 +216,7 @@ await foreach (TableEntity region in repo.EnumerateAsync())
 > Install-Package Devlooped.TableStorage
 ```
 
-There is also a source-only version, if you want to avoid an additional assembly:
+There is also a source-only version, if you want to avoid an additional assembly dependency:
 
 ```
 > Install-Package Devlooped.TableStorage.Source
@@ -179,8 +234,16 @@ namespace Devlooped
     public partial interface ITablePartition<T> { }
     public partial class TableRepository { }
     public partial class TableRepository<T> { }
+    public partial class AttributedTableRepository<T> { }
+    public partial class DocumentRepository { }
+    public partial class DocumentRepository<T> { }
+    public partial class AttributedDocumentRepository<T> { }
+    public partial interface IDocumentSerializer { }
+    public partial interface IBinaryDocumentSerializer { }
+    public partial interface IStringDocumentSerializer { }
     public partial class TablePartition { }
     public partial class TablePartition<T> { }
+    public partial class DocumentPartition { }
 
     // Perhaps make the attributes visible too if you use them?
     public partial class TableAttribute { }

assets/img/entity.png

@@ -0,0 +1,10 @@
+<Project>
+
+  <ItemGroup>
+    <Compile Update="@(Compile -> WithMetadataValue('NuGetPackageId', 'Devlooped.TableStorage.Bson.Source'))">
+      <Visible>false</Visible>
+      <Link>Devlooped\TableStorage.Bson\%(Filename)%(Extension)</Link>
+    </Compile>
+  </ItemGroup>
+  
+</Project>

readme.md

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <AssemblyName>Devlooped.TableStorage.Bson.Source</AssemblyName>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IsPackable>true</IsPackable>
+    <PackBuildOutput>false</PackBuildOutput>
+    <PackCompile>true</PackCompile>
+    <Description>A source-only BSON binary serializer for use with document-based repositories.
+
+Usage:
+
+  var repo = DocumentRepository.Create&lt;Product&gt;(storageAccount, serializer: BsonDocumentSerializer.Default);
+    </Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NuGetizer" Version="0.7.0" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="all" />
+    <PackageReference Include="Newtonsoft.Json.Bson" Version="1.0.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\TableStorage.Source\TableStorage.Source.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\TableStorage.Bson\**\*.cs" Exclude="..\TableStorage.Bson\Visibility.cs;..\TableStorage.Bson\obj\**\*.cs;" />
+    <None Update="Devlooped.TableStorage.Bson.Source.targets" PackFolder="build" />
+  </ItemGroup>
+  
+</Project>

src/TableStorage.Bson.Source/Devlooped.TableStorage.Bson.Source.targets

@@ -0,0 +1,44 @@
+//<auto-generated/>
+#nullable enable
+using System.IO;
+using Newtonsoft.Json;
+
+namespace Devlooped
+{
+    /// <summary>
+    /// Default implementation of <see cref="IBinaryDocumentSerializer"/> which 
+    /// uses Newtonsoft.Json implementation of BSON for serialization.
+    /// </summary>
+    partial class BsonDocumentSerializer : IBinaryDocumentSerializer
+    {
+        static readonly JsonSerializer serializer = new JsonSerializer();
+
+        /// <summary>
+        /// Default instance of the serializer.
+        /// </summary>
+        public static IDocumentSerializer Default { get; } = new BsonDocumentSerializer();
+
+        /// <inheritdoc />
+        public T? Deserialize<T>(byte[] data)
+        {
+            if (data.Length == 0)
+                return default;
+
+            using var mem = new MemoryStream(data);
+            using var reader = new Newtonsoft.Json.Bson.BsonDataReader(mem);
+            return (T?)serializer.Deserialize<T>(reader);
+        }
+
+        /// <inheritdoc />
+        public byte[] Serialize<T>(T value)
+        {
+            if (value == null)
+                return new byte[0];
+
+            using var mem = new MemoryStream();
+            using var writer = new Newtonsoft.Json.Bson.BsonDataWriter(mem);
+            serializer.Serialize(writer, value);
+            return mem.ToArray();
+        }
+    }
+}

src/TableStorage.Bson.Source/TableStorage.Bson.Source.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <AssemblyName>Devlooped.TableStorage.Bson</AssemblyName>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IsPackable>true</IsPackable>
+    <Description>A BSON binary serializer for use with document-based repositories.
+
+Usage:
+
+  var repo = DocumentRepository.Create&lt;Product&gt;(storageAccount, serializer: BsonDocumentSerializer.Default);
+    </Description>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NuGetizer" Version="0.7.0" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="all" />
+    <PackageReference Include="Newtonsoft.Json.Bson" Version="1.0.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\TableStorage\TableStorage.csproj" />
+  </ItemGroup>
+
+</Project>

src/TableStorage.Bson/BsonDocumentSerializer.cs

@@ -0,0 +1,6 @@
+//<auto-generated/>
+namespace Devlooped
+{
+    // Sets default visibility when using compiled version, where everything is public
+    public partial class BsonDocumentSerializer { }
+}