Skip to content

Commit d6e17ed

Browse files
RoelantVosRoelantVos
committed
- Fixed configuration bug which prevented some of the content files not to be copied (Sat RI pattern, test data pattern)
- Updated version to 1.6.3 - Added DIRECT and Table Generation patterns
1 parent 3783ea3 commit d6e17ed

6 files changed

+2532
-6570
lines changed

‎Virtual_EDW/LoadPatterns/loadPatternCollection.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@
1313
"loadPatternFilePath": "loadPatternHubInsertInto.handlebars",
1414
"loadPatternNotes": "Pattern to load a Hub view into a physical table (persistence)."
1515
},
16+
{
17+
"loadPatternName": "Data Vault Hub 002 Insert Into Pattern with DIRECT wrapper",
18+
"loadPatternType": "CoreBusinessConcept",
19+
"loadPatternConnectionKey": "IntegrationConnectionInternalId",
20+
"loadPatternFilePath": "loadPatternHubInsertIntoDirectWrapper.handlebars",
21+
"loadPatternNotes": "Pattern to load a Hub view into a physical table (persistence), integrated with the DIRECT framework."
22+
},
1623
{
1724
"loadPatternName": "Data Vault Hub 003 Delta Select From Pattern",
1825
"loadPatternType": "CoreBusinessConcept",
@@ -26,6 +33,13 @@
2633
"loadPatternConnectionKey": "PsaConnectionInternalId",
2734
"loadPatternFilePath": "loadPatternHubView_DBT_Snowflake.handlebars",
2835
"loadPatternNotes": "Standard Hub pattern - display the distinct list of Business Keys for a Core Business Concept."
36+
},
37+
{
38+
"loadPatternName": "Data Vault Hub 005 Generate Tables for DIRECT wrapper",
39+
"loadPatternType": "CoreBusinessConcept",
40+
"loadPatternConnectionKey": "IntegrationConnectionInternalId",
41+
"loadPatternFilePath": "loadPatternHubGenerateTables.handlebars",
42+
"loadPatternNotes": "Creating tables using the DIRECT standard conventions."
2943
},
3044
{
3145
"loadPatternName": "Data Vault Link 001 View Pattern",

‎Virtual_EDW/LoadPatterns/loadPatternHubGenerateTables.Handlebars‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{{#each dataObjectMappings}}{{#if @first}}
2+
--
3+
-- Table creation statement for {{targetDataObject.name}}
4+
-- Generated at {{now}}
5+
--
6+
7+
IF OBJECT_ID('[{{targetDataObject.name}}]', 'U') IS NOT NULL
8+
DROP TABLE [{{targetDataObject.name}}];
9+
10+
CREATE TABLE [{{targetDataObject.name}}]
11+
(
12+
[{{#each businessKeys}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] binary(16) NOT NULL,
13+
[MODULE_INSTANCE_ID] [int] NOT NULL,
14+
[{{../metadataConfiguration.loadDateTimeAttribute}}] [datetime2](7) NOT NULL,
15+
[{{../metadataConfiguration.recordSourceAttribute}}] [nvarchar](100) NOT NULL,
16+
{{#each businessKeys}}{{#each businessKeyComponentMapping}}[{{targetDataItem.name}}] [nvarchar](100) NOT NULL{{#unless @last}},{{/unless}}
17+
{{/each}}{{/each}}
18+
){{/if}}
19+
{{/each}}

‎Virtual_EDW/LoadPatterns/loadPatternHubInsertIntoDirectWrapper.Handlebars‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
{{#each dataObjectMappings}}
2+
GO
3+
--
4+
-- Hub ETL procedure for {{targetDataObject.name}}, coming from {{sourceDataObjects.0.name}}
5+
-- Generated at {{now}}
6+
--
7+
8+
CREATE OR ALTER PROCEDURE [usp_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}]
9+
@BatchInstanceId INT = 0,
10+
@Debug VARCHAR(1) = 'N'
11+
AS
12+
13+
/* For debugging purposes */
14+
DECLARE @Reset VARCHAR(10) = 'N';
15+
DECLARE @Module_id INT;
16+
17+
-- Module Registration
18+
INSERT INTO [900_Direct_Framework].[omd].MODULE (MODULE_CODE, MODULE_DESCRIPTION, MODULE_TYPE, DATA_OBJECT_SOURCE, DATA_OBJECT_TARGET, AREA_CODE, FREQUENCY_CODE, INACTIVE_INDICATOR)
19+
SELECT *
20+
FROM
21+
(
22+
VALUES ('m_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}', 'Data logistics from {{sourceDataObjects.0.name}} to {{targetDataObject.name}}', 'SQL', 'None', '[{{targetDataObject.dataObjectConnection.extensions.0.value}}].[{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]','INT','Continuous', 'N')
23+
) AS refData( MODULE_CODE, MODULE_DESCRIPTION, MODULE_TYPE, DATA_OBJECT_SOURCE, DATA_OBJECT_TARGET, AREA_CODE, FREQUENCY_CODE, INACTIVE_INDICATOR)
24+
WHERE NOT EXISTS
25+
(
26+
SELECT NULL
27+
FROM [900_Direct_Framework].[omd].MODULE module
28+
WHERE module.MODULE_CODE = refData.MODULE_CODE
29+
);
30+
31+
-- Clean up and reset
32+
SELECT @Module_id = module_id FROM [900_Direct_Framework].[omd].module WHERE module_code = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}'
33+
34+
IF @Reset = 'Y'
35+
BEGIN
36+
DELETE FROM [900_Direct_Framework].[omd].event_log WHERE MODULE_INSTANCE_ID IN (SELECT MODULE_INSTANCE_ID FROM [900_Direct_Framework].[omd].module_instance WHERE MODULE_ID=@Module_id)
37+
DELETE FROM [900_Direct_Framework].[omd].source_control WHERE MODULE_INSTANCE_ID IN (SELECT MODULE_INSTANCE_ID FROM [900_Direct_Framework].[omd].module_instance WHERE MODULE_ID=@Module_id)
38+
DELETE FROM [900_Direct_Framework].[omd].module_instance WHERE MODULE_ID=@Module_id
39+
DELETE FROM [900_Direct_Framework].[omd].batch_module where module_id = @Module_id
40+
END
41+
42+
/* End of debugging block */
43+
44+
45+
/* Start new Module Instance */
46+
DECLARE @ModuleCode VARCHAR(255) = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}'
47+
48+
BEGIN
49+
-- Module Execution Return Value
50+
DECLARE @QueryResult VARCHAR(MAX);
51+
52+
-- Create Module Instance
53+
DECLARE @ModuleInstanceId INT
54+
EXEC [900_Direct_Framework].[omd].[CreateModuleInstance]
55+
@ModuleCode = @ModuleCode,
56+
@Debug = @Debug,
57+
@BatchInstanceId = @BatchInstanceId, -- The Batch Instance Id, if the Module is run from a Batch.
58+
@ModuleInstanceId = @ModuleInstanceId OUTPUT;
59+
60+
-- Module Evaluation
61+
DECLARE @ProcessIndicator VARCHAR(10);
62+
EXEC [900_Direct_Framework].[omd].[ModuleEvaluation]
63+
@ModuleInstanceId = @ModuleInstanceId,
64+
@Debug = @Debug,
65+
@ProcessIndicator = @ProcessIndicator OUTPUT;
66+
67+
IF @Debug = 'Y'
68+
PRINT @ProcessIndicator;
69+
70+
IF @ProcessIndicator NOT IN ('Abort','Cancel') -- These are end-states for the process.
71+
BEGIN TRY
72+
73+
/*
74+
Main ETL block
75+
*/
76+
77+
INSERT INTO [{{targetDataObject.name}}]
78+
(
79+
[{{#each businessKeys}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}],{{#each businessKeys}}{{#if @first}}{{#each businessKeyComponentMapping}}
80+
[{{targetDataItem.name}}],{{/each}}{{/if}}{{/each}}
81+
[MODULE_INSTANCE_ID],
82+
[{{../metadataConfiguration.loadDateTimeAttribute}}],
83+
[{{../metadataConfiguration.recordSourceAttribute}}]
84+
)
85+
SELECT
86+
hub_view.[{{#each businessKeys}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}],{{#each businessKeys}}{{#if @first}}{{#each businessKeyComponentMapping}}
87+
hub_view.[{{targetDataItem.name}}],{{/each}}{{/if}}{{/each}}
88+
@ModuleInstanceId AS [MODULE_INSTANCE_ID],
89+
hub_view.[{{../metadataConfiguration.loadDateTimeAttribute}}],
90+
hub_view.[{{../metadataConfiguration.recordSourceAttribute}}]
91+
FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.0.value}}].[{{../metadataConfiguration.vdwSchemaName}}].[{{targetDataObject.name}}] hub_view
92+
LEFT OUTER JOIN [{{targetDataObject.name}}] hub_table
93+
ON hub_view.[{{#each businessKeys}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] = hub_table.[{{#each businessKeys}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}]
94+
WHERE hub_table.[{{#each businessKeys}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] IS NULL
95+
96+
DECLARE @RowCountInsert INT= @@ROWCOUNT;
97+
98+
IF @Debug = 'Y'
99+
PRINT CONVERT(VARCHAR(10),@RowCountInsert)+' rows inserted';
100+
101+
/*
102+
Wrap up
103+
*/
104+
105+
IF @Debug = 'Y'
106+
PRINT 'Success pathway';
107+
108+
-- Module Success
109+
EXEC [900_Direct_Framework].[omd].[UpdateModuleInstance]
110+
@ModuleInstanceId = @ModuleInstanceId,
111+
@RowCountSelect = 0,
112+
@RowCountInsert = @RowCountInsert,
113+
@Debug = @Debug,
114+
@EventCode = 'Success'
115+
116+
SET @QueryResult = 'Success';
117+
118+
END TRY
119+
120+
BEGIN CATCH
121+
IF @Debug = 'Y'
122+
PRINT 'Failure pathway';
123+
124+
-- Module Failure
125+
EXEC [900_Direct_Framework].[omd].[UpdateModuleInstance]
126+
@ModuleInstanceId = @ModuleInstanceId,
127+
@Debug = @Debug,
128+
@EventCode = 'Failure';
129+
130+
SET @QueryResult = 'Failure';
131+
132+
-- Logging
133+
DECLARE
134+
@EventDetail VARCHAR(4000) = ERROR_MESSAGE(),
135+
@EventReturnCode int = ERROR_NUMBER();
136+
137+
EXEC [900_Direct_Framework].[omd].[InsertIntoEventLog]
138+
@ModuleInstanceId = @ModuleInstanceId,
139+
@EventDetail = @EventDetail,
140+
@EventReturnCode = @EventReturnCode;
141+
142+
THROW
143+
END CATCH
144+
145+
ELSE
146+
SET @QueryResult = @ProcessIndicator;
147+
148+
-- End of procedure
149+
END
150+
{{/each}}

‎Virtual_EDW/LoadPatterns/loadPatternPersistentStagingFullOuterJoin.handlebarsBackup_20201124225448‎

Lines changed: 0 additions & 128 deletions
This file was deleted.

‎Virtual_EDW/Virtual_Data_Warehouse.csproj‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -345,8 +345,18 @@
345345
<Content Include="Examples\STG_PROFILER_PLAN.json">
346346
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
347347
</Content>
348-
<None Include="LoadPatterns\loadPatternSatelliteReferentialIntegrity.Handlebars" />
349-
<None Include="LoadPatterns\loadPatternStagingTestData.Handlebars" />
348+
<Content Include="LoadPatterns\loadPatternSatelliteReferentialIntegrity.Handlebars">
349+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
350+
</Content>
351+
<Content Include="LoadPatterns\loadPatternStagingTestData.Handlebars">
352+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
353+
</Content>
354+
<Content Include="LoadPatterns\loadPatternHubGenerateTables.Handlebars">
355+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
356+
</Content>
357+
<Content Include="LoadPatterns\loadPatternHubInsertIntoDirectWrapper.Handlebars">
358+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
359+
</Content>
350360
<None Include="Properties\app.manifest" />
351361
<Content Include="Resources\Ravos-logo-small.ico" />
352362
<Content Include="Schema\interfaceDataWarehouseAutomationMetadata.json">

0 commit comments

Comments
 (0)