Skip to content

Commit ce04332

Browse files
RoelantVosRoelantVos
committed
Simple export function
1 parent 015460d commit ce04332

File tree

4 files changed

+170
-19
lines changed

4 files changed

+170
-19
lines changed

‎DataWarehouseAutomation/DataWarehouseAutomation.sln

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RunDwhAutomation", "RunDwhA
5252
EndProject
5353
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SchemaJsonConverter", "SchemaJsonConverter\SchemaJsonConverter.csproj", "{07214B80-37A9-4137-B0F3-109FA0D0926A}"
5454
EndProject
55+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Export", "Export\Export.csproj", "{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}"
56+
EndProject
5557
Global
5658
GlobalSection(SolutionConfigurationPlatforms) = preSolution
5759
Debug|Any CPU = Debug|Any CPU
@@ -78,6 +80,10 @@ Global
7880
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Debug|Any CPU.Build.0 = Debug|Any CPU
7981
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.ActiveCfg = Release|Any CPU
8082
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.Build.0 = Release|Any CPU
83+
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
84+
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
85+
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
86+
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.Build.0 = Release|Any CPU
8187
EndGlobalSection
8288
GlobalSection(SolutionProperties) = preSolution
8389
HideSolutionNode = FALSE

‎DataWarehouseAutomation/Export/Export.csproj

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net7.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="DataWarehouseAutomation" Version="1.3.4" />
12+
</ItemGroup>
13+
14+
</Project>

‎DataWarehouseAutomation/Export/Program.cs

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
using System.Dynamic;
2+
using DataWarehouseAutomation;
3+
using HandlebarsDotNet;
4+
using Newtonsoft.Json;
5+
6+
var inputMetadataDirectory = @"D:\Git_Repos\jarvis-TEAM-metadata\Development\TEAM\Metadata";
7+
var outputMetadataDirectory = @"D:\Git_Repos\";
8+
9+
var exceptionList = new List<string>
10+
{
11+
"VDW_Samples_TEAM_Attribute_Mapping.json",
12+
"Development_TEAM_Attribute_Mapping.json",
13+
"sample_TEAM_Attribute_Mapping.json"
14+
};
15+
16+
List<DataItemMappingTuple> exportOutput = new List<DataItemMappingTuple>();
17+
18+
foreach (string file in Directory.EnumerateFiles(inputMetadataDirectory, "*.json", SearchOption.TopDirectoryOnly))
19+
{
20+
if (!exceptionList.Contains(Path.GetFileName(file)))
21+
{
22+
try
23+
{
24+
Console.WriteLine(file);
25+
var json = File.ReadAllText(file);
26+
var deserializedMapping = JsonConvert.DeserializeObject<DataWarehouseAutomation.DataObjectMappingList>(json);
27+
28+
foreach (var dataObjectMapping in deserializedMapping.DataObjectMappings)
29+
{
30+
var sourceDataObjectRaw = dataObjectMapping.SourceDataObjects.FirstOrDefault();
31+
32+
if (dataObjectMapping.DataItemMappings != null)
33+
{
34+
foreach (var dataItemMapping in dataObjectMapping.DataItemMappings)
35+
{
36+
var sourceDataItemRaw = dataItemMapping.SourceDataItems.FirstOrDefault();
37+
38+
var localDataItemMappingTuple = new DataItemMappingTuple
39+
{
40+
SourceDataObject = sourceDataObjectRaw.name,
41+
SourceDataItem = sourceDataItemRaw.name,
42+
TargetDataObject = dataObjectMapping.TargetDataObject.Name,
43+
TargetDataItem = dataItemMapping.TargetDataItem.Name
44+
};
45+
46+
if (!exportOutput.Contains(localDataItemMappingTuple))
47+
{
48+
exportOutput.Add(localDataItemMappingTuple);
49+
}
50+
}
51+
}
52+
}
53+
}
54+
catch (Exception exception)
55+
{
56+
Console.WriteLine($"Issue: "+exception.Message);
57+
}
58+
}
59+
}
60+
61+
// Export to file
62+
using (StreamWriter writer = new StreamWriter(outputMetadataDirectory+"TEAM-export.csv"))
63+
{
64+
foreach (var exportRow in exportOutput)
65+
{
66+
writer.WriteLine($"{exportRow.SourceDataObject},{exportRow.SourceDataItem},{exportRow.TargetDataObject},{exportRow.TargetDataItem}");
67+
}
68+
}
69+
70+
// Finish the application.
71+
Console.ReadKey();
72+
73+
internal class DataItemMappingTuple
74+
{
75+
internal string SourceDataObject
76+
{
77+
get;
78+
set;
79+
}
80+
81+
internal string SourceDataItem
82+
{
83+
get;
84+
set;
85+
}
86+
87+
internal string TargetDataObject
88+
{
89+
get;
90+
set;
91+
}
92+
93+
internal string TargetDataItem
94+
{
95+
get;
96+
set;
97+
}
98+
}

‎DataWarehouseAutomation/SchemaJsonConverter/Program.cs

Lines changed: 52 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -126,14 +126,6 @@
126126
jsonObjectDataObjectMapping.Remove("mappingName");
127127
jsonObjectDataObjectMapping.Add("name", mappingNameNode);
128128

129-
var getName = jsonObjectDataObjectMapping.TryGetPropertyValue("name", out var mappingNameJsonNode).ToString();
130-
131-
// Add the mapping name as a 'name' to the list of mappings, only once.
132-
if (jsonObject["dataObjectMappings"].AsArray().IndexOf(dataObjectMapping) == 0)
133-
{
134-
jsonObject["name"] = mappingNameJsonNode.ToString();
135-
}
136-
137129
// Rename the mapping classifications.
138130
var mappingClassificationsNode = jsonObjectDataObjectMapping["mappingClassifications"];
139131
jsonObjectDataObjectMapping.Remove("mappingClassifications");
@@ -142,6 +134,18 @@
142134
// Update the classifications
143135
UpdateClassifications(jsonObjectDataObjectMapping, classificationGuids);
144136

137+
var getName = jsonObjectDataObjectMapping.TryGetPropertyValue("name", out var mappingNameJsonNode).ToString();
138+
var getClassification = jsonObjectDataObjectMapping.TryGetPropertyValue("classifications", out var mappingClassificationJsonNode).ToString();
139+
140+
var copyOfMappingClassificationNode = JsonNode.Parse(mappingClassificationJsonNode.ToJsonString());
141+
142+
// Add the mapping name as a 'name', and the classification, to the list of mappings, only once.
143+
if (jsonObject["dataObjectMappings"].AsArray().IndexOf(dataObjectMapping) == 0)
144+
{
145+
jsonObject["name"] = mappingNameJsonNode.ToString();
146+
jsonObject["classifications"] = copyOfMappingClassificationNode;
147+
}
148+
145149
// Rename the business key definitions.
146150
var businessKeyDefinitionsNode = jsonObjectDataObjectMapping["businessKeys"];
147151
jsonObjectDataObjectMapping.Remove("businessKeys");
@@ -183,23 +187,26 @@
183187

184188
// Data Items.
185189
var dataItems = new List<JsonObject>();
186-
foreach (var dataItem in dataObjectJsonObject["dataItems"].AsArray())
190+
if (dataObjectJsonObject["dataItems"] != null)
187191
{
188-
var dataItemJsonObject = JsonNode.Parse(dataItem.ToJsonString()).AsObject();
192+
foreach (var dataItem in dataObjectJsonObject["dataItems"].AsArray())
193+
{
194+
var dataItemJsonObject = JsonNode.Parse(dataItem.ToJsonString()).AsObject();
189195

190-
// Type must be first.
191-
dataItemJsonObject.Add("dataItemType", "dataItem");
196+
// Type must be first.
197+
dataItemJsonObject.Add("dataItemType", "dataItem");
192198

193-
// Ensure each item has a GUID
194-
AddGuid(dataItemJsonObject, dataItemGuids, "name");
199+
// Ensure each item has a GUID
200+
AddGuid(dataItemJsonObject, dataItemGuids, "name");
195201

196-
// Re-add unchanged properties to manage order.
197-
AddUnchangedDataItemProperties(dataItemJsonObject);
202+
// Re-add unchanged properties to manage order.
203+
AddUnchangedDataItemProperties(dataItemJsonObject);
198204

199-
// Replace properties with newer names (upgrade).
200-
ReplaceDataObjectProperties(dataItemJsonObject);
205+
// Replace properties with newer names (upgrade).
206+
ReplaceDataObjectProperties(dataItemJsonObject);
201207

202-
dataItems.Add(dataItemJsonObject);
208+
dataItems.Add(dataItemJsonObject);
209+
}
203210
}
204211

205212
foreach (var dataItem in dataItems)
@@ -773,6 +780,32 @@ void UpdateClassifications(JsonObject jsonObject, Dictionary<string, Guid> objec
773780
var classificationJsonObject = JsonNode.Parse(classification.ToJsonString()).AsObject();
774781
var getClassification = classificationJsonObject.TryGetPropertyValue("classification", out var classificationValue).ToString();
775782

783+
// Rename to phyiscal model object classifications.
784+
string newClassificationValue = classificationValue.ToString() switch
785+
{
786+
"Source" => "Source",
787+
"Core Business Concept" => "Hub",
788+
"CoreBusinessConcept" => "Hub",
789+
"Context" => "Satellite",
790+
"Natural Business Relationship" => "Link",
791+
"NaturalBusinessRelationship" => "Link",
792+
"Natural Business Relationship Context" => "Link-Satellite",
793+
"NaturalBusinessRelationshipContext" => "Link-Satellite",
794+
"Natural Business Relationship Context Driving Key" => "Link-Satellite Driving Key",
795+
"NaturalBusinessRelationshipContextDrivingKey" => "Link-Satellite Driving Key",
796+
"Person" => "Conceptual",
797+
"Place" => "Conceptual",
798+
"Event" => "Physical",
799+
_ => "Unknown"
800+
};
801+
802+
if (newClassificationValue != "Unknown")
803+
{
804+
classificationJsonObject["classification"] = newClassificationValue;
805+
classificationValue = newClassificationValue;
806+
}
807+
808+
// Add a group.
776809
string groupValue = classificationValue.ToString() switch
777810
{
778811
"Source" => "Solution Layer",

0 commit comments

Comments
 (0)