Skip to content

Commit 27012c0

Browse files
RoelantVosRoelantVos
committed
Updates of samples to match TEAM v1.6.2.
1 parent d37b2bc commit 27012c0

File tree

4 files changed

+118
-39
lines changed

4 files changed

+118
-39
lines changed

‎Virtual_EDW/Examples/DIM_CUSTOMER.json‎

Lines changed: 77 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,44 @@
66
"expiryDateTimeAttribute": "LOAD_END_DATETIME",
77
"eventDateTimeAttribute": "EVENT_DATETIME",
88
"recordChecksumAttribute": "HASH_FULL_RECORD",
9-
"etlProcessAttribute": "ETL_INSERT_RUN_ID",
9+
"etlProcessAttribute": "MODULE_INSTANCE_ID",
1010
"sourceRowIdAttribute": "SOURCE_ROW_ID"
1111
},
1212
"generationSpecificMetadata": {
1313
"selectedDataObject": "DIM_CUSTOMER",
14-
"generationDateTime": "2020-11-17T14:18:42.9293307+10:00"
14+
"generationDateTime": "2021-01-25T17:29:58.4876393+10:00"
1515
},
1616
"dataObjectMappings": [
1717
{
1818
"mappingName": "DIM_CUSTOMER",
1919
"mappingClassifications": [
2020
{
21-
"classification": "Point-in-Time / Dimension"
21+
"classification": "Presentation"
2222
}
2323
],
2424
"sourceDataObjects": [
2525
{
2626
"name": "HUB_CUSTOMER",
27+
"dataItems": [
28+
{
29+
"name": "CUSTOMER_ID"
30+
}
31+
],
32+
"dataObjectConnection": {
33+
"dataConnectionString": "Integration",
34+
"extensions": [
35+
{
36+
"key": "database",
37+
"value": "200_Integration_Layer",
38+
"description": "database name"
39+
},
40+
{
41+
"key": "schema",
42+
"value": "dbo",
43+
"description": "schema name"
44+
}
45+
]
46+
},
2747
"dataObjectClassifications": [
2848
{
2949
"classification": "CoreBusinessConcept"
@@ -32,11 +52,6 @@
3252
},
3353
{
3454
"name": "SAT_CUSTOMER",
35-
"dataObjectClassifications": [
36-
{
37-
"classification": "Context"
38-
}
39-
],
4055
"dataItems": [
4156
{
4257
"name": "DATE_OF_BIRTH"
@@ -47,30 +62,65 @@
4762
{
4863
"name": "SUBURB"
4964
}
65+
],
66+
"dataObjectConnection": {
67+
"dataConnectionString": "Integration",
68+
"extensions": [
69+
{
70+
"key": "database",
71+
"value": "200_Integration_Layer",
72+
"description": "database name"
73+
},
74+
{
75+
"key": "schema",
76+
"value": "dbo",
77+
"description": "schema name"
78+
}
79+
]
80+
},
81+
"dataObjectClassifications": [
82+
{
83+
"classification": "Context"
84+
}
5085
]
5186
},
5287
{
5388
"name": "SAT_CUSTOMER_ADDITIONAL_DETAILS",
54-
"dataObjectClassifications": [
89+
"dataItems": [
5590
{
56-
"classification": "Context"
91+
"name": "CONTACT_NUMBER"
5792
}
5893
],
59-
"dataItems": [
94+
"dataObjectConnection": {
95+
"dataConnectionString": "Integration",
96+
"extensions": [
97+
{
98+
"key": "database",
99+
"value": "200_Integration_Layer",
100+
"description": "database name"
101+
},
102+
{
103+
"key": "schema",
104+
"value": "dbo",
105+
"description": "schema name"
106+
}
107+
]
108+
},
109+
"dataObjectClassifications": [
60110
{
61-
"name": "CONTACT_NUMBER"
111+
"classification": "Context"
62112
}
63113
]
64114
}
65115
],
66116
"targetDataObject": {
67117
"name": "DIM_CUSTOMER",
68118
"dataObjectConnection": {
69-
"dataConnectionString": "PSA",
119+
"dataConnectionString": "Presentation",
70120
"extensions": [
71121
{
72122
"key": "database",
73-
"value": "150_Persistent_Staging_Area",
123+
"value": "300_Presentation_Layer",
74124
"description": "database name"
75125
},
76126
{
@@ -82,6 +132,16 @@
82132
}
83133
},
84134
"dataItemMappings": [
135+
{
136+
"sourceDataItems": [
137+
{
138+
"name": "CUSTOMER_ID"
139+
}
140+
],
141+
"targetDataItem": {
142+
"name": "CUSTOMER_ID"
143+
}
144+
},
85145
{
86146
"sourceDataItems": [
87147
{
@@ -129,15 +189,15 @@
129189
{
130190
"sourceDataItems": [
131191
{
132-
"name": "CustomerID"
192+
"name": "CUSTOMER_SK"
133193
}
134194
],
135195
"targetDataItem": {
136-
"name": "CUSTOMER_ID"
196+
"name": "CUSTOMER_SK"
137197
}
138198
}
139199
],
140-
"surrogateKey": "CUSTOMER_HSH"
200+
"surrogateKey": "DIM_CUSTOMER_SK"
141201
}
142202
],
143203
"enabled": true

‎Virtual_EDW/Form_Main.cs‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Security.Permissions;
99
using System.Text;
1010
using System.Threading;
11+
using System.Threading.Tasks;
1112
using System.Windows.Forms;
1213
using DataWarehouseAutomation;
1314
using Newtonsoft.Json;
@@ -285,9 +286,11 @@ public void RunFileWatcher()
285286
FileSystemWatcher watcher = new FileSystemWatcher
286287
{
287288
Path = VdwConfigurationSettings.VdwInputPath,
288-
Filter = "*.json"
289+
Filter = "*.json",
289290
};
290291

292+
var timeOut = Task.Delay(500);
293+
291294
// Add event handlers.
292295
watcher.Changed += FileWatcherOnChanged;
293296
watcher.Created += FileWatcherOnChanged;
@@ -314,9 +317,25 @@ private void FileWatcherOnError(object source, ErrorEventArgs e)
314317
// Define the event handlers.
315318
private void FileWatcherOnChanged(object source, FileSystemEventArgs e)
316319
{
317-
InformUser($"File {e.Name} was modified in {VdwConfigurationSettings.VdwInputPath}.", EventTypes.Information);
318-
319-
CreateCustomTabPages();
320+
// Make sure the damn thing only fires once.
321+
var localFileSystemWatcher = (FileSystemWatcher) source;
322+
try
323+
{
324+
325+
localFileSystemWatcher.EnableRaisingEvents = false;
326+
327+
InformUser($"File {e.Name} was modified in {VdwConfigurationSettings.VdwInputPath}.", EventTypes.Information);
328+
329+
CreateCustomTabPages();
330+
331+
}
332+
333+
finally
334+
{
335+
localFileSystemWatcher.EnableRaisingEvents = true;
336+
}
337+
338+
320339
}
321340

322341

‎Virtual_EDW/LoadPatterns/loadPatternCollection.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
"loadPatternConnectionKey": "IntegrationConnectionInternalId",
2020
"loadPatternFilePath": "loadPatternHubInsertIntoDirectWrapper.handlebars",
2121
"loadPatternNotes": "Pattern to load a Hub view into a physical table (persistence), integrated with the DIRECT framework."
22-
},
22+
},
2323
{
2424
"loadPatternName": "Data Vault Hub 003 Delta Select From Pattern",
2525
"loadPatternType": "CoreBusinessConcept",
@@ -33,7 +33,7 @@
3333
"loadPatternConnectionKey": "PsaConnectionInternalId",
3434
"loadPatternFilePath": "loadPatternHubView_DBT_Snowflake.handlebars",
3535
"loadPatternNotes": "Standard Hub pattern - display the distinct list of Business Keys for a Core Business Concept."
36-
},
36+
},
3737
{
3838
"loadPatternName": "Data Vault Hub 005 Generate Tables for DIRECT wrapper",
3939
"loadPatternType": "CoreBusinessConcept",
@@ -92,7 +92,7 @@
9292
},
9393
{
9494
"loadPatternName": "Dimension / PIT 001 Select Pattern",
95-
"loadPatternType": "Point-in-Time / Dimension",
95+
"loadPatternType": "Presentation",
9696
"loadPatternConnectionKey": "PsaConnectionInternalId",
9797
"loadPatternFilePath": "loadPatternDimSelect.handlebars",
9898
"loadPatternNotes": "Taken from earlier VDW versions, demonstrating one possible DIM / PIT template."

‎Virtual_EDW/LoadPatterns/loadPatternDimSelect.Handlebars‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66

77
SELECT
88
[PIT_EFFECTIVE_DATETIME],{{#each sourceDataObjects}}{{stringcompare dataObjectClassifications.0.classification 'CoreBusinessConcept'}}{{sourceData}}
9-
LEAD(PIT_EFFECTIVE_DATETIME,1,'9999-12-31') OVER (PARTITION BY [{{name}}.{{../../businessKeys.0.surrogateKey}}] ORDER BY PIT_EFFECTIVE_DATETIME ASC) AS PIT_EXPIRY_DATETIME,
10-
COALESCE([{{name}}.{{../../businessKeys.0.surrogateKey}}],0x00000000000000000000000000000000) AS [{{name}}.{{../../businessKeys.0.surrogateKey}}],
9+
LEAD(PIT_EFFECTIVE_DATETIME,1,'9999-12-31') OVER (PARTITION BY [{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}] ORDER BY PIT_EFFECTIVE_DATETIME ASC) AS PIT_EXPIRY_DATETIME,
10+
COALESCE([{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}],0x00000000000000000000000000000000) AS [{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}],
1111
{{else}}
12-
COALESCE([{{name}}.{{../../businessKeys.0.surrogateKey}}],0x00000000000000000000000000000000) AS [{{name}}.{{../../businessKeys.0.surrogateKey}}],
12+
COALESCE([{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}],0x00000000000000000000000000000000) AS [{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}],
1313
{{#each dataItems}}
1414
[{{../name}}.{{name}}],
1515
{{/each}}
@@ -20,16 +20,16 @@ FROM
2020
(
2121
SELECT
2222
*,{{#each sourceDataObjects}}{{stringcompare dataObjectClassifications.0.classification 'CoreBusinessConcept'}}{{sourceData}}
23-
LAG(ATTRIBUTE_CHECKSUM, 1, 0x00000000000000000000000000000000) OVER(PARTITION BY [{{name}}.{{../../businessKeys.0.surrogateKey}}] ORDER BY PIT_EFFECTIVE_DATETIME ASC) AS PREVIOUS_ATTRIBUTE_CHECKSUM
23+
LAG(ATTRIBUTE_CHECKSUM, 1, 0x00000000000000000000000000000000) OVER(PARTITION BY [{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}] ORDER BY PIT_EFFECTIVE_DATETIME ASC) AS PREVIOUS_ATTRIBUTE_CHECKSUM
2424
{{/stringcompare}}
2525
{{/each}}
2626
FROM
2727
(
2828
SELECT
2929
*,{{#each sourceDataObjects}}{{stringcompare dataObjectClassifications.0.classification 'CoreBusinessConcept'}}{{sourceData}}
30-
ISNULL(RTRIM(CONVERT(VARCHAR(100),[{{name}}.{{../../businessKeys.0.surrogateKey}}])),'NA')+'|'+
30+
ISNULL(RTRIM(CONVERT(VARCHAR(100),[{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}])),'NA')+'|'+
3131
{{else}}
32-
ISNULL(RTRIM(CONVERT(VARCHAR(100),[{{name}}.{{../../businessKeys.0.surrogateKey}}])),'NA')+'|'+
32+
ISNULL(RTRIM(CONVERT(VARCHAR(100),[{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}])),'NA')+'|'+
3333
{{#each dataItems}}
3434
ISNULL(RTRIM(CONVERT(VARCHAR(100),[{{../name}}.{{name}}])),'NA')+'|'+
3535
{{/each}}
@@ -40,9 +40,9 @@ FROM
4040
FROM
4141
(
4242
SELECT{{#each sourceDataObjects}}{{stringcompare dataObjectClassifications.0.classification 'CoreBusinessConcept'}}{{sourceData}}
43-
{{name}}.{{../../businessKeys.0.surrogateKey}} AS [{{name}}.{{../../businessKeys.0.surrogateKey}}],
43+
{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}} AS [{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}],
4444
{{else}}
45-
{{name}}.{{../../businessKeys.0.surrogateKey}} AS [{{name}}.{{../../businessKeys.0.surrogateKey}}],
45+
{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}} AS [{{name}}.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}],
4646
{{name}}.{{../../../metadataConfiguration.loadDateTimeAttribute}} AS [{{name}}.{{../../../metadataConfiguration.loadDateTimeAttribute}}],
4747
{{#each dataItems}}
4848
{{../name}}.{{name}} AS [{{../name}}.{{name}}],
@@ -53,29 +53,29 @@ FROM
5353
FROM
5454
(
5555
SELECT
56-
{{businessKeys.0.surrogateKey}},
56+
{{businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}},
5757
PIT_EFFECTIVE_DATETIME,
58-
LEAD(PIT_EFFECTIVE_DATETIME,1,'9999-12-31') OVER (PARTITION BY {{businessKeys.0.surrogateKey}} ORDER BY PIT_EFFECTIVE_DATETIME ASC) AS PIT_EXPIRY_DATETIME
58+
LEAD(PIT_EFFECTIVE_DATETIME,1,'9999-12-31') OVER (PARTITION BY {{businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}} ORDER BY PIT_EFFECTIVE_DATETIME ASC) AS PIT_EXPIRY_DATETIME
5959
FROM
6060
(
6161
{{#each sourceDataObjects}}
6262
{{stringcompare dataObjectClassifications.0.classification 'CoreBusinessConcept'}}
63-
SELECT {{../../businessKeys.0.surrogateKey}}, CONVERT(DATETIME2(7), '1900-01-01') AS PIT_EFFECTIVE_DATETIME FROM {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
63+
SELECT {{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}, CONVERT(DATETIME2(7), '1900-01-01') AS PIT_EFFECTIVE_DATETIME FROM {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
6464
UNION
65-
SELECT {{../../businessKeys.0.surrogateKey}}, {{../../../metadataConfiguration.loadDateTimeAttribute}} AS PIT_EFFECTIVE_DATETIME FROM {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
65+
SELECT {{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}, {{../../../metadataConfiguration.loadDateTimeAttribute}} AS PIT_EFFECTIVE_DATETIME FROM {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
6666
{{else}}
67-
SELECT {{../../businessKeys.0.surrogateKey}}, {{../../../metadataConfiguration.loadDateTimeAttribute}} AS PIT_EFFECTIVE_DATETIME FROM {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
67+
SELECT {{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}, {{../../../metadataConfiguration.loadDateTimeAttribute}} AS PIT_EFFECTIVE_DATETIME FROM {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
6868
{{/stringcompare}}
6969
{{#unless @last}} UNION{{/unless}}
7070
{{/each}}
7171
) PIT
7272
) TimeRanges
7373
{{#each sourceDataObjects}}{{stringcompare dataObjectClassifications.0.classification 'CoreBusinessConcept'}}{{sourceData}}
7474
INNER JOIN {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
75-
ON TimeRanges.{{../../businessKeys.0.surrogateKey}} = {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}.[{{../../businessKeys.0.surrogateKey}}]
75+
ON TimeRanges.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}} = {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}.[{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}]
7676
{{else}}
7777
LEFT OUTER JOIN {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}
78-
ON TimeRanges.{{../../businessKeys.0.surrogateKey}} = {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}.[{{../../businessKeys.0.surrogateKey}}]
78+
ON TimeRanges.{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}} = {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}.[{{../../businessKeys.0.businessKeyComponentMapping.0.targetDataItem.name}}]
7979
AND {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}.{{../../../metadataConfiguration.loadDateTimeAttribute}} <= TimeRanges.PIT_EFFECTIVE_DATETIME
8080
AND {{../../../metadataConfiguration.vdwSchemaName}}.{{name}}.{{../../../metadataConfiguration.expiryDateTimeAttribute}} >= TimeRanges.PIT_EXPIRY_DATETIME
8181
{{/stringcompare}}

0 commit comments

Comments
 (0)