Skip to content

Commit cf44102

Browse files
RoelantVosRoelantVos
committed
Added filewatcher.
1 parent 496e0af commit cf44102

File tree

2 files changed

+198
-47
lines changed

2 files changed

+198
-47
lines changed

‎Virtual_EDW/Form_Main.cs‎

Lines changed: 70 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,6 @@ public FormMain()
109109
var comboItem = comboBoxEnvironments.Items.Cast<KeyValuePair<string, TeamWorkingEnvironment>>().FirstOrDefault(item => item.Value.Equals(FormBase.VdwConfigurationSettings.ActiveEnvironment));
110110
comboBoxEnvironments.SelectedItem = comboItem;
111111

112-
// Start monitoring the configuration directories for file changes
113-
// RunFileWatcher(); DISABLED FOR NOW - FIRES 2 EVENTS!!
114-
115112
richTextBoxInformationMain.AppendText("Application initialised - welcome to the Virtual Data Warehouse! \r\n\r\n");
116113

117114
checkBoxGenerateInDatabase.Checked = false;
@@ -139,6 +136,9 @@ public FormMain()
139136
localCustomTabPage.setSaveOutputFileFlag(true);
140137
}
141138

139+
// Start monitoring the configuration directories for file changes
140+
RunFileWatcher();
141+
142142
startUpIndicator = false;
143143
}
144144

@@ -263,29 +263,8 @@ public void CreateLoadPatternCollectionDataGrid()
263263

264264
private void GridAutoLayoutLoadPatternCollection()
265265
{
266-
//dataGridViewLoadPatternCollection.AutoSizeColumnsMode = DataGridViewAutoSizeColumnsMode.Fill;
267-
////Set the auto-size based on (the contents of) all cells in each column.
268-
//for (var i = 0; i < dataGridViewLoadPatternCollection.Columns.Count - 1; i++)
269-
//{
270-
// dataGridViewLoadPatternCollection.Columns[i].AutoSizeMode = DataGridViewAutoSizeColumnMode.AllCells;
271-
// int localWidth = dataGridViewLoadPatternCollection.Columns[i].Width;
272-
//}
273-
274-
//// Choose one column to be used to fill out the grid
275-
//if (dataGridViewLoadPatternCollection.Columns.Count > 0)
276-
//{
277-
// dataGridViewLoadPatternCollection.Columns[dataGridViewLoadPatternCollection.Columns.Count - 1].AutoSizeMode = DataGridViewAutoSizeColumnMode.Fill;
278-
//}
279-
280-
//// Disable the auto size again (to enable manual resizing).
281-
//for (var i = 0; i < dataGridViewLoadPatternCollection.Columns.Count - 1; i++)
282-
//{
283-
// int columnWidth = dataGridViewLoadPatternCollection.Columns[i].Width;
284-
// dataGridViewLoadPatternCollection.Columns[i].AutoSizeMode = DataGridViewAutoSizeColumnMode.None;
285-
// dataGridViewLoadPatternCollection.Columns[i].Width = columnWidth;
286-
//}
287266
dataGridViewLoadPatternCollection.AutoSizeColumnsMode = DataGridViewAutoSizeColumnsMode.AllCells;
288-
//dataGridViewLoadPatternCollection.Columns[3].AutoSizeMode = DataGridViewAutoSizeColumnMode.Fill;
267+
289268
dataGridViewLoadPatternCollection.Columns[dataGridViewLoadPatternCollection.ColumnCount - 1].AutoSizeMode = DataGridViewAutoSizeColumnMode.Fill;
290269

291270
// Disable the auto size again (to enable manual resizing).
@@ -394,34 +373,44 @@ private void SetDatabaseConnections()
394373
}
395374

396375
[PermissionSet(SecurityAction.Demand, Name = "FullTrust")]
397-
public static void RunFileWatcher()
376+
public void RunFileWatcher()
398377
{
399378
// Create a new FileSystemWatcher and set its properties.
400-
FileSystemWatcher watcher = new FileSystemWatcher();
401-
//watcher.Path = (GlobalParameters.ConfigurationPath + GlobalParameters.ConfigfileName);
402-
403-
watcher.Path = VdwConfigurationSettings.TeamEnvironmentFilePath;
404-
405-
/* Watch for changes in LastAccess and LastWrite times, and
406-
the renaming of files or directories. */
407-
watcher.NotifyFilter = NotifyFilters.LastWrite;
408-
// Only watch text files.
409-
watcher.Filter = GlobalParameters.TeamConfigurationFileName;
379+
FileSystemWatcher watcher = new FileSystemWatcher
380+
{
381+
Path = VdwConfigurationSettings.VdwInputPath,
382+
Filter = "*.json"
383+
};
410384

411385
// Add event handlers.
412-
watcher.Changed += OnChanged;
413-
// watcher.Created += new FileSystemEventHandler(OnChanged);
414-
// watcher.Deleted += new FileSystemEventHandler(OnChanged);
386+
watcher.Changed += FileWatcherOnChanged;
387+
watcher.Created += FileWatcherOnChanged;
388+
watcher.Deleted += FileWatcherOnChanged;
389+
watcher.Error += FileWatcherOnError;
415390

391+
watcher.SynchronizingObject = this;
416392
// Begin watching.
417393
watcher.EnableRaisingEvents = true;
418394
}
419395

396+
private void FileWatcherOnError(object source, ErrorEventArgs e)
397+
{
398+
if (e.GetException().GetType() == typeof(InternalBufferOverflowException))
399+
{
400+
InformUser("File System Watcher internal buffer overflow.",EventTypes.Error);
401+
}
402+
else
403+
{
404+
InformUser($"Watched directory {VdwConfigurationSettings.VdwInputPath} not accessible by the system.", EventTypes.Error);
405+
}
406+
}
407+
420408
// Define the event handlers.
421-
private static void OnChanged(object source, FileSystemEventArgs e)
409+
private void FileWatcherOnChanged(object source, FileSystemEventArgs e)
422410
{
423-
// Specify what is done when a file is changed, created, or deleted.
424-
MessageBox.Show("File changed");
411+
InformUser($"File {e.Name} was modified in {VdwConfigurationSettings.VdwInputPath}.", EventTypes.Information);
412+
413+
CreateCustomTabPages();
425414
}
426415

427416

@@ -491,7 +480,7 @@ public void ThreadProcAbout()
491480
}
492481
}
493482

494-
#region Multi-threading delegates for text boxes
483+
#region Multi-threading delegates
495484

496485
/// <summary>
497486
/// Delegate to update the main information textbox.
@@ -512,8 +501,40 @@ private void SetTextMain(string text)
512501
}
513502
}
514503

504+
505+
delegate void CallBackAddCustomTabPage(TabPage tabPage);
506+
507+
private void AddCustomTabPage(TabPage tabPage)
508+
{
509+
if (tabControlMain.InvokeRequired)
510+
{
511+
var d = new CallBackAddCustomTabPage(AddCustomTabPage);
512+
Invoke(d, tabPage);
513+
}
514+
else
515+
{
516+
tabControlMain.TabPages.Add(tabPage);
517+
518+
}
519+
}
520+
521+
delegate void CallBackRemoveCustomTabPage(TabPage tabPage);
522+
523+
private void RemoveCustomTabPage(TabPage tabPage)
524+
{
525+
if (tabControlMain.InvokeRequired)
526+
{
527+
var d = new CallBackRemoveCustomTabPage(RemoveCustomTabPage);
528+
Invoke(d, tabPage);
529+
}
530+
else
531+
{
532+
tabControlMain.TabPages.Remove(tabPage);
533+
}
534+
}
535+
515536
#endregion
516-
537+
517538
#region Background worker
518539

519540
// This event handler deals with the results of the background operation.
@@ -738,11 +759,12 @@ internal class LocalPattern
738759
internal string notes { get; set; }
739760
internal Dictionary<string,VDW_DataObjectMappingList> itemList { get; set; }
740761
}
762+
741763

742764
internal void InformUser(string text, EventTypes eventType)
743765
{
744766
VdwConfigurationSettings.VdwEventLog.Add(Event.CreateNewEvent(eventType, $"{text}"));
745-
richTextBoxInformationMain.AppendText(text + "\r\n");
767+
SetTextMain(text + "\r\n");
746768
}
747769

748770
/// <summary>
@@ -958,7 +980,7 @@ internal void CreateCustomTabPages()
958980
else
959981
{
960982
// Remove the Tab Page from the Tab Control
961-
tabControlMain.Controls.Remove((customTabPage));
983+
RemoveCustomTabPage(customTabPage);
962984
}
963985
}
964986

@@ -973,7 +995,8 @@ internal void CreateCustomTabPages()
973995
localCustomTabPage.OnClearMainText += ClearMainInformationTextBox;
974996

975997
localCustomTabPageList.Add(localCustomTabPage);
976-
tabControlMain.TabPages.Add(localCustomTabPage);
998+
999+
AddCustomTabPage(localCustomTabPage);
9771000
}
9781001

9791002
// Work around issue related to incorrectly enabled metadata extract

‎Virtual_EDW/LoadPatterns/loadPatternPersistentStagingFullOuterJoin.handlebarsBackup_20201124225448‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
{{#each dataObjectMappings}}
2+
--
3+
-- Landing <=> PSA full outer join query for {{targetDataObject.name}}
4+
-- Generated at {{now}}
5+
--
6+
7+
PRINT 'Working on {{targetDataObject.name}}';
8+
PRINT GETDATE();
9+
10+
WITH STG_CTE AS
11+
(
12+
SELECT
13+
[{{../metadataConfiguration.loadDateTimeAttribute}}],
14+
[{{../metadataConfiguration.eventDateTimeAttribute}}],
15+
[{{../metadataConfiguration.recordSourceAttribute}}],
16+
{{#each dataItemMappings}}
17+
[{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}],
18+
{{/each}}
19+
HASHBYTES('MD5',
20+
{{#each dataItemMappings}}
21+
ISNULL(RTRIM(CONVERT(NVARCHAR(100),[{{sourceDataItems.0.name}}])), 'N/A') + '#~!'{{#unless @last}} +{{/unless}}
22+
{{/each}}
23+
) AS [{{../metadataConfiguration.recordChecksumAttribute}}]
24+
FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.0.value}}].[{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}]
25+
),
26+
PSA_CTE AS
27+
(
28+
SELECT
29+
A.[{{../metadataConfiguration.recordChecksumAttribute}}] AS [{{../metadataConfiguration.recordChecksumAttribute}}],
30+
A.[{{../metadataConfiguration.loadDateTimeAttribute}}] AS [{{../metadataConfiguration.loadDateTimeAttribute}}],
31+
A.[{{../metadataConfiguration.eventDateTimeAttribute}}] AS [{{../metadataConfiguration.eventDateTimeAttribute}}],
32+
A.[{{../metadataConfiguration.recordSourceAttribute}}] AS [{{../metadataConfiguration.recordSourceAttribute}}],
33+
{{#each dataItemMappings}}
34+
A.[{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}
35+
{{/each}}
36+
FROM [{{targetDataObject.dataObjectConnection.extensions.0.value}}].[{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] A
37+
JOIN
38+
(
39+
SELECT {{#each businessKeys}} {{#each businessKeyComponentMapping}}
40+
[{{sourceDataItems.0.name}}], {{/each}} {{/each}}
41+
MAX({{../metadataConfiguration.loadDateTimeAttribute}}) AS MAX_{{../metadataConfiguration.loadDateTimeAttribute}}
42+
FROM [{{targetDataObject.dataObjectConnection.extensions.0.value}}].[{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] B
43+
GROUP BY {{#each businessKeys}} {{#each businessKeyComponentMapping}}
44+
[{{sourceDataItems.0.name}}] {{#unless @last}},{{/unless}}{{/each}} {{/each}}
45+
) B ON {{#each businessKeys}} {{#each businessKeyComponentMapping}}
46+
A.[{{sourceDataItems.0.name}}] = B.[{{sourceDataItems.0.name}}] AND {{/each}} {{/each}}
47+
A.{{../metadataConfiguration.loadDateTimeAttribute}} = B.MAX_{{../metadataConfiguration.loadDateTimeAttribute}}
48+
WHERE {{../metadataConfiguration.changeDataCaptureAttribute}} != 'Delete'
49+
), MAIN_CTE AS
50+
(
51+
SELECT
52+
CASE
53+
WHEN STG_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL
54+
THEN PSA_CTE.[{{../metadataConfiguration.recordChecksumAttribute}}]
55+
ELSE STG_CTE.[{{../metadataConfiguration.recordChecksumAttribute}}]
56+
END AS [{{../metadataConfiguration.recordChecksumAttribute}}],
57+
CASE
58+
WHEN STG_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL THEN 'Delete'
59+
WHEN PSA_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL THEN 'Insert'
60+
WHEN STG_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NOT NULL
61+
AND PSA_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NOT NULL
62+
AND STG_CTE.[{{../metadataConfiguration.recordChecksumAttribute}}] != PSA_CTE.[{{../metadataConfiguration.recordChecksumAttribute}}] THEN 'Change' ELSE 'No Change'
63+
END AS {{../metadataConfiguration.changeDataCaptureAttribute}},
64+
ROW_NUMBER() OVER
65+
(ORDER BY
66+
{{#each businessKeys}}{{#each businessKeyComponentMapping}}CASE WHEN STG_CTE.[{{sourceDataItems.0.name}}] IS NULL THEN PSA_CTE.[{{sourceDataItems.0.name}}] ELSE STG_CTE.[{{sourceDataItems.0.name}}] END{{#unless @last}},{{/unless}}{{/each}}{{/each}}
67+
) AS {{../metadataConfiguration.sourceRowIdAttribute}},
68+
CASE
69+
WHEN STG_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL
70+
THEN PSA_CTE.[{{../metadataConfiguration.loadDateTimeAttribute}}]
71+
ELSE STG_CTE.[{{../metadataConfiguration.loadDateTimeAttribute}}]
72+
END AS [{{../metadataConfiguration.loadDateTimeAttribute}}],
73+
CASE
74+
WHEN STG_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL
75+
THEN PSA_CTE.[{{../metadataConfiguration.eventDateTimeAttribute}}]
76+
ELSE STG_CTE.[{{../metadataConfiguration.eventDateTimeAttribute}}]
77+
END AS [{{../metadataConfiguration.eventDateTimeAttribute}}],
78+
CASE
79+
WHEN STG_CTE.{{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL
80+
THEN PSA_CTE.[{{../metadataConfiguration.recordSourceAttribute}}]
81+
ELSE STG_CTE.[{{../metadataConfiguration.recordSourceAttribute}}]
82+
END AS [{{../metadataConfiguration.recordSourceAttribute}}],
83+
{{#each dataItemMappings}}
84+
CASE
85+
WHEN STG_CTE.{{#each ../businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}[{{sourceDataItems.0.name}}]{{/if}}{{/each}}{{/each}} IS NULL
86+
THEN PSA_CTE.[{{sourceDataItems.0.name}}]
87+
ELSE STG_CTE.[{{sourceDataItems.0.name}}]
88+
END AS [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}}
89+
{{/each}}
90+
FROM STG_CTE
91+
FULL OUTER JOIN PSA_CTE ON {{#each businessKeys}} {{#each businessKeyComponentMapping}}
92+
PSA_CTE.[{{sourceDataItems.0.name}}] = STG_CTE.[{{sourceDataItems.0.name}}]{{#unless @last}} AND{{/unless}}{{/each}} {{/each}}
93+
WHERE
94+
(
95+
CASE
96+
WHEN {{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}STG_CTE.[{{sourceDataItems.0.name}}] IS NULL THEN 'Delete'{{/if}}{{/each}}{{/each}}
97+
WHEN {{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}PSA_CTE.[{{sourceDataItems.0.name}}] IS NULL THEN 'Insert'{{/if}}{{/each}}{{/each}}
98+
WHEN {{#each businessKeys}}{{#each businessKeyComponentMapping}}{{#if @first}}PSA_CTE.[{{sourceDataItems.0.name}}] IS NOT NULL AND PSA_CTE.{{sourceDataItems.0.name}} IS NOT NULL AND STG_CTE.[{{../../../metadataConfiguration.recordChecksumAttribute}}] != PSA_CTE.[{{../../../metadataConfiguration.recordChecksumAttribute}}] THEN 'Change'{{/if}}{{/each}}{{/each}}
99+
ELSE 'No Change'
100+
END
101+
) != 'No Change'
102+
)
103+
INSERT INTO [{{targetDataObject.dataObjectConnection.extensions.0.value}}].[{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]
104+
(
105+
[{{../metadataConfiguration.etlProcessAttribute}}],
106+
[{{../metadataConfiguration.loadDateTimeAttribute}}],
107+
[{{../metadataConfiguration.eventDateTimeAttribute}}],
108+
[{{../metadataConfiguration.recordSourceAttribute}}],
109+
[{{../metadataConfiguration.sourceRowIdAttribute}}],
110+
[{{../metadataConfiguration.changeDataCaptureAttribute}}],
111+
[{{../metadataConfiguration.recordChecksumAttribute}}],
112+
{{#each dataItemMappings}}
113+
[{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}}
114+
{{/each}}
115+
)
116+
SELECT
117+
-1 AS [{{../metadataConfiguration.etlProcessAttribute}}],
118+
[{{../metadataConfiguration.loadDateTimeAttribute}}],
119+
[{{../metadataConfiguration.eventDateTimeAttribute}}],
120+
[{{../metadataConfiguration.recordSourceAttribute}}],
121+
[{{../metadataConfiguration.sourceRowIdAttribute}}],
122+
[{{../metadataConfiguration.changeDataCaptureAttribute}}],
123+
[{{../metadataConfiguration.recordChecksumAttribute}}],
124+
{{#each dataItemMappings}}
125+
[{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}}
126+
{{/each}}
127+
FROM MAIN_CTE;
128+
{{/each}}

0 commit comments

Comments
 (0)