Merge pull request #29 from data-solution-automation-engine/dev-2024-06

Dev 2024 06 - 2.1.0 version

Author: RoelantVos

DataWarehouseAutomation/DataWarehouseAutomation.sln

@@ -13,21 +13,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		Images\Ravos-logo.png = Images\Ravos-logo.png
 	EndProjectSection
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Validation", "Test_Project\Validation.csproj", "{23CF4A46-B17F-4642-B949-02E061868BBC}"
-	ProjectSection(ProjectDependencies) = postProject
-		{F0458983-9646-42E6-9E18-90AD91288B40} = {F0458983-9646-42E6-9E18-90AD91288B40}
-	EndProjectSection
-EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample Metadata", "Sample Metadata", "{242F4BD0-68F9-4193-A329-826E755C6B08}"
 	ProjectSection(SolutionItems) = preProject
 		Sample_Metadata\myFirstMapping.json = Sample_Metadata\myFirstMapping.json
 		Sample_Metadata\sampleBasic.json = Sample_Metadata\sampleBasic.json
 		Sample_Metadata\sampleBasicWithExtensions.json = Sample_Metadata\sampleBasicWithExtensions.json
 		Sample_Metadata\sampleCalculation.json = Sample_Metadata\sampleCalculation.json
 		Sample_Metadata\sampleCustomFunctions.json = Sample_Metadata\sampleCustomFunctions.json
-		Sample_Metadata\sampleDataVaultHub.json = Sample_Metadata\sampleDataVaultHub.json
 		Sample_Metadata\sampleFreeForm.json = Sample_Metadata\sampleFreeForm.json
-		Sample_Metadata\sampleJsonStagingWithPsaDetails.json = Sample_Metadata\sampleJsonStagingWithPsaDetails.json
 		Sample_Metadata\sampleMultipleDataItemMappings.json = Sample_Metadata\sampleMultipleDataItemMappings.json
 		Sample_Metadata\sampleSimpleDDL.json = Sample_Metadata\sampleSimpleDDL.json
 		Sample_Metadata\sampleSourceQuery.json = Sample_Metadata\sampleSourceQuery.json
@@ -46,13 +39,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample Templates", "Sample
 		Sample_Templates\TemplateSampleSourceQuery.Handlebars = Sample_Templates\TemplateSampleSourceQuery.Handlebars
 	EndProjectSection
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Examples", "Example_Project\Examples.csproj", "{F77AC518-6300-4927-8775-B92CDF15CC8E}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RunDwhAutomation", "RunDwhAutomation\RunDwhAutomation.csproj", "{62C992C6-370D-4591-B54E-2F61E2D8B638}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SchemaJsonConverter", "SchemaJsonConverter\SchemaJsonConverter.csproj", "{07214B80-37A9-4137-B0F3-109FA0D0926A}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Export", "Export\Export.csproj", "{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Validation", "Validation\Validation.csproj", "{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Examples", "Examples\Examples.csproj", "{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -64,14 +57,6 @@ Global
 		{F0458983-9646-42E6-9E18-90AD91288B40}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F0458983-9646-42E6-9E18-90AD91288B40}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F0458983-9646-42E6-9E18-90AD91288B40}.Release|Any CPU.Build.0 = Release|Any CPU
-		{23CF4A46-B17F-4642-B949-02E061868BBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{23CF4A46-B17F-4642-B949-02E061868BBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{23CF4A46-B17F-4642-B949-02E061868BBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{23CF4A46-B17F-4642-B949-02E061868BBC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F77AC518-6300-4927-8775-B92CDF15CC8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F77AC518-6300-4927-8775-B92CDF15CC8E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F77AC518-6300-4927-8775-B92CDF15CC8E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F77AC518-6300-4927-8775-B92CDF15CC8E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{62C992C6-370D-4591-B54E-2F61E2D8B638}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{62C992C6-370D-4591-B54E-2F61E2D8B638}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{62C992C6-370D-4591-B54E-2F61E2D8B638}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -80,10 +65,14 @@ Global
 		{07214B80-37A9-4137-B0F3-109FA0D0926A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.Build.0 = Release|Any CPU
-		{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

DataWarehouseAutomation/DataWarehouseAutomation/DataWarehouseAutomation.csproj

@@ -4,14 +4,14 @@
     <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
-    <Version>2.0.3</Version>
+    <Version>2.1.0</Version>
     <Authors>Roelant Vos</Authors>
     <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
     <Title>Data Warehouse Automation</Title>
-    <PackageProjectUrl>https://roelantvos.com/blog/interface-for-data-warehouse-automation/</PackageProjectUrl>
+    <PackageProjectUrl>https://github.com/data-solution-automation-engine/data-warehouse-automation-metadata-schema</PackageProjectUrl>
     <PackageIcon>Ravos-logo.png</PackageIcon>
     <PackageReadmeFile>index.md</PackageReadmeFile>
-    <RepositoryUrl>https://github.com/RoelantVos/data-warehouse-automation-metadata-schema</RepositoryUrl>
+    <RepositoryUrl>https://github.com/data-solution-automation-engine/data-warehouse-automation-metadata-schema</RepositoryUrl>
     <PackageTags>data warehouse; etl; data integration; data warehouse automation</PackageTags>
     <Description>The generic schema for Data Warehouse Automation covers the efforts towards defining a standard interface to exchange Data Warehouse Automation and ETL generation metadata.
 
@@ -35,6 +35,22 @@ The interface itself is a Json schema definition (JSDL) with examples on how to
     <PackageReference Include="Handlebars.Net" Version="2.1.4" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Newtonsoft.Json.Schema" Version="3.0.15" />
+    <PackageReference Include="Roslynator.Analyzers" Version="4.12.4">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Roslynator.CodeFixes" Version="4.12.4">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.12.4">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Roslynator.Refactorings" Version="4.12.4">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
 </Project>

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/BusinessKeyDefinition.cs

@@ -1,8 +1,8 @@
 namespace DataWarehouseAutomation.DwaModel;
 
 /// <summary>
-/// A Business Key, which consists of one or more components (column mappings) and has its own surrogate key.
-/// A Business Key is a special column, or combination of columns, that is defined separately outside of regular data item mappings.
+/// A business key definition is a special object that is defined as an optional property of a data object mapping.
+/// In other words, the business key definition can be a part of describing the relationship between a source data object and a target data object.
 /// </summary>
 public class BusinessKeyDefinition
 {
@@ -14,16 +14,18 @@ public class BusinessKeyDefinition
     public string? Id { get; set; }
 
     /// <summary>
-    /// The optional name of the Business Key Definition.
+    /// The optional name of the business key definition.
     /// </summary>
     [JsonPropertyName("name")]
     public string? Name { get; set; }
 
     /// <summary>
-    /// Items that define the Business Key e.g. the collection of columns for a Business Key.
+    /// Items that define the Business Key e.g. the collection of columns for a business key.
+    /// The integer value defines the order of the components in the business key definition.
+    /// The order is significant, and is recorded as part of the tuple definition.
     /// </summary>
-    [JsonPropertyName("businessKeyComponentMappings")]
-    public List<DataItemMapping> BusinessKeyComponentMappings { get; set; } = new();
+    [JsonPropertyName("businessKeyComponents")]
+    public List<BusinessKeyComponents>? BusinessKeyComponents { get; set; }
 
     /// <summary>
     /// An optional label for the end result e.g. the target business key attribute.
@@ -33,14 +35,14 @@ public class BusinessKeyDefinition
     public string? SurrogateKey { get; set; }
 
     /// <summary>
-    /// Free-form and optional classification for the Business Key for use in generation logic (evaluation).
+    /// Free-form and optional classification for the business key for use in generation logic (evaluation).
     /// </summary>
     [JsonPropertyName("classifications")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
     public List<DataClassification>? Classifications { get; set; }
 
     /// <summary>
-    /// The collection of extension Key/Value pairs.
+    /// The collection of extension key/value pairs.
     /// </summary>
     [JsonPropertyName("extensions")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/Cardinality.cs

@@ -0,0 +1,86 @@
+namespace DataWarehouseAutomation.DwaModel;
+
+/// <summary>
+/// This object captures the cardinality and ordinality of a relationship.
+/// Cardinality refers to the uniqueness of data values contained in a column (attribute) of a database table.
+/// It defines the number of occurrences of one entity that are associated with the number of occurrences of another entity through a relationship.
+/// </summary>
+public class Cardinality
+{
+    /// <summary>
+    /// Optional identifier as a string value to allow various identifier approaches.
+    /// </summary>
+    [JsonPropertyName("id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public string? Id { get; set; }
+
+    /// <summary>
+    /// Optional information to name a certain cardinality construct.
+    /// For example one-to-one, one-to-many, or many-to-many.
+    /// E.g. one-to-one could be defined as {"fromRange": {"min": "1", "max": "1"}, "toRange": {"min": "1", "max": "1"}}.
+    /// </summary>
+    [JsonPropertyName("name")]
+    public string? Name { get; set; }
+
+    /// <summary>
+    /// The 'from' component in the cardinality, e.g. the '1' in 1 to many.
+    /// </summary>
+    [JsonPropertyName("fromRange")]
+    public CardinalityRange? FromRange { get; set; } = new CardinalityRange { Min = "1", Max = "1" };
+
+    /// <summary>
+    /// The 'to' component in the cardinality, e.g. the 'many' in 1 to many.
+    /// </summary>
+    [JsonPropertyName("toRange")]
+    public CardinalityRange? ToRange { get; set; } = new CardinalityRange { Min = "1", Max = "N" };
+
+    /// <summary>
+    /// Free-form and optional classification for the Cardinality for use in generation logic (evaluation).
+    /// </summary>
+    [JsonPropertyName("classifications")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public List<DataClassification>? Classifications { get; set; }
+
+    /// <summary>
+    /// The collection of extension Key/Value pairs.
+    /// </summary>
+    [JsonPropertyName("extensions")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public List<Extension>? Extensions { get; set; }
+
+    /// <summary>
+    /// Free-format notes.
+    /// </summary>
+    [JsonPropertyName("notes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public string? Notes { get; set; }
+
+    #region Methods
+    /// <summary>
+    /// Use this method to assert if two Cardinalities are the same, based on their Ids.
+    /// </summary>
+    /// <param name="obj"></param>
+    /// <returns>True if the Cardinalities are the same, based on their Ids</returns>
+    public override bool Equals(object? obj)
+    {
+        var other = obj as Cardinality;
+        return other?.Id == Id;
+    }
+
+    /// <summary>
+    /// Override to get a hash value that represents the identifier.
+    /// </summary>
+    /// <returns>A 32-bit signed integer hash code</returns>
+    public override int GetHashCode() => (Id?.GetHashCode()) ?? 0;
+
+    /// <summary>
+    /// String override so that the object returns its value ('Name').
+    /// When an instance of this class is passed to a method that expects a string, the ToString() method will be called implicitly to convert the object to a string, and the value of the "Name" property will be returned.
+    /// </summary>
+    /// <returns>The Name</returns>
+    public override string ToString()
+    {
+        return Name ?? string.Empty;
+    }
+    #endregion
+}

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/CardinalityRange.cs

@@ -0,0 +1,15 @@
+namespace DataWarehouseAutomation.DwaModel;
+
+/// <summary>
+/// The possible range for a from/to component for the cardinality.
+/// For example "min": "1", "max": "N"
+/// This way, you can define "at least 1 to many". Or "0 or 1 to 1".
+/// </summary>
+public class CardinalityRange
+{
+    [JsonPropertyName("min")]
+    public string? Min { get; set; }
+
+    [JsonPropertyName("max")]
+    public string? Max { get; set; }
+}

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/DataItem.cs

@@ -1,5 +1,9 @@
 namespace DataWarehouseAutomation.DwaModel;
 
+/// <summary>
+/// Data items belong to data objects or data object queries. This means that a given data item always has a parent object.
+/// They describe the individual elements, such as the columns in a table or headers in a file.
+/// </summary>
 public class DataItem : IDataItem
 {
     /// <summary>
@@ -14,7 +18,7 @@ public class DataItem : IDataItem
     public string Name { get; set; } = "NewDataItem";
 
     /// <summary>
-    /// The data object to which the data item belongs. This can be used to construct fully qualified names.
+    /// The <see cref="IDataObject"/> to which the data item belongs. This can be used to construct fully qualified names.
     /// </summary>
     [JsonPropertyName("dataObject")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/DataItemMapping.cs

@@ -1,7 +1,7 @@
 namespace DataWarehouseAutomation.DwaModel;
 
 /// <summary>
-/// The individual column-to-column mapping.
+/// The individual <see cref="IDataItem"/> to <see cref="IDataItem"/>, column-to-column mapping.
 /// </summary>
 public class DataItemMapping
 {
@@ -13,7 +13,7 @@ public class DataItemMapping
     public string? Id { get; set; }
 
     [JsonPropertyName("sourceDataItems")]
-    public List<IDataItem> SourceDataItems { get; set; } = new();
+    public List<IDataItem> SourceDataItems { get; set; } = [];
 
     [JsonPropertyName("targetDataItem")]
     public IDataItem TargetDataItem { get; set; } = new DataItem() { Name = "newTargetDataItem" };

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/DataItemQuery.cs

@@ -16,8 +16,9 @@ public class DataItemQuery : IDataItem
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
     public string Name { get; set; } = string.Empty;
 
-    // convenience property for parent, derived on json load, never stored in the json file
-    // Can be either a DataObject or a DataQuery when the DataQuery is a DataItem level thing
+    /// <summary>
+    /// The <see cref="IDataObject"/> to which the data item belongs. This can be used to construct fully qualified names.
+    /// </summary>
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
     public IDataObject? DataObject { get; set; }
 

DataWarehouseAutomation/DataWarehouseAutomation/DwaModel/DataObject.cs

@@ -2,8 +2,9 @@
 
 /// <summary>
 /// The definition of a data set, file, or table.
-/// The Data Object can be the 'source' or 'target' in a Data Object Mapping.
-/// A Data Object which acts as target in one mapping, can be a source in another mapping, building up the data logistics lineage.
+/// The Data Object can be the 'source' or 'target' in a <see cref="DataObjectMapping"/>.
+/// A Data Object which acts as target in one mapping, can be a source in another mapping,
+/// building up the data logistics lineage.
 /// </summary>
 public class DataObject : IMetadata, IDataObject
 {
@@ -35,7 +36,24 @@ public class DataObject : IMetadata, IDataObject
     public DataConnection? DataConnection { get; set; }
 
     /// <summary>
-    /// Free-form and optional classification for the Data Object for use in ETL generation logic (evaluation).
+    /// The definition of the Business Key(s) for the Data Object.
+    /// Being able to record the business key definition 
+    /// This serves multiple purposes, but one of them is to support defining a series of business key definitions against the source data object, and reuse these across different data object mappings.
+    /// The order is stored as well, because in some cases the order of keys is meaningful.
+    /// </summary>
+    [JsonPropertyName("businessKeyDefinitions")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public List<BusinessKeyDefinitions>? BusinessKeyDefinitions { get; set; }
+
+    /// <summary>
+    /// Any relationship to other data objects.
+    /// </summary>
+    [JsonPropertyName("relationships")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public List<Relationships>? Relationships { get; set; }
+
+    /// <summary>
+    /// Free-form and optional classification for the Data Object.
     /// </summary>
     [JsonPropertyName("classifications")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]