update all remaining post deploy scripts

use merge to allow more elegant and sophisticated data logistics for the reference sets

Author: sjohansson

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.AREA.sql

@@ -3,16 +3,29 @@
  * Reference data insert and update script
  * DIRECT Framework v2.0
  *
+ * Reference metadata table AREA stores areas and layers information.
+ *
+ * This script is used to insert and update reference data on deployment.
+ * Any bespoke event types added manually to the target will be retained,
+ * as long as the keys differ.
+ *
+ * To maintain a clean CI/CD process, consider using this script to manage
+ * all reference data for event types.
+ *
  * [omd_metadata].[AREA]
  *
  ******************************************************************************/
+
 SET NOCOUNT ON;
 
-INSERT INTO [omd_metadata].[AREA]
-SELECT [AREA_CODE], [LAYER_CODE], [AREA_DESCRIPTION]
-FROM (
-  /* AREA_CODE, LAYER_CODE, AREA_DESCRIPTION */
-  VALUES
+DECLARE @tblMerge TABLE(
+  [AREA_CODE]             NVARCHAR (100)  NOT NULL PRIMARY KEY CLUSTERED,
+  [LAYER_CODE]            NVARCHAR (100)  NOT NULL,
+  [AREA_DESCRIPTION]      NVARCHAR (4000) NULL
+);
+
+INSERT INTO @tblMerge([AREA_CODE], [LAYER_CODE], [AREA_DESCRIPTION])
+VALUES
   (N'HELPER',       N'Presentation',  N'The Helper Area'),
   (N'INT',          N'Integration',   N'The Base Integration Area'),
   (N'INTPR',        N'Integration',   N'The Derived Integration Area'),
@@ -22,11 +35,22 @@ FROM (
   (N'PSA',          N'Staging',       N'The Persistent Staging Area'),
   (N'STG',          N'Staging',       N'The Staging Area of the Staging Layer'),
   (N'SYNC',         N'Staging',       N'Synchronization of the production History Area of the Staging Layer for build and test')
- ) AS refData(AREA_CODE, LAYER_CODE, AREA_DESCRIPTION)
-WHERE NOT EXISTS (
-  SELECT NULL
-  FROM [omd_metadata].[AREA] a
-  WHERE a.AREA_CODE = refData.AREA_CODE
-  );
+
+ MERGE [omd_metadata].[AREA] AS TARGET
+  USING @tblMerge AS src
+      ON  TARGET.[AREA_CODE] = src.[AREA_CODE]
+
+  WHEN MATCHED THEN
+      UPDATE
+      SET      [LAYER_CODE] = src.[LAYER_CODE],
+               [AREA_DESCRIPTION] = src.[AREA_DESCRIPTION]
+
+  WHEN NOT MATCHED THEN
+      INSERT  ([AREA_CODE]
+              ,[LAYER_CODE]
+              ,[AREA_DESCRIPTION])
+      VALUES  ([AREA_CODE]
+              ,[LAYER_CODE]
+              ,[AREA_DESCRIPTION]);
 
 GO

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.EVENT_TYPE.sql

@@ -23,22 +23,22 @@ DECLARE @tblMerge TABLE(
 
 INSERT INTO @tblMerge([EVENT_TYPE_CODE], [EVENT_TYPE_CODE_DESCRIPTION])
 VALUES
- (N'1', N'Infrastructure error.'),
- (N'2', N'Internal data integration process error or system generated event.'),
- (N'3', N'Custom exception handling that has been implemented in code (Error Bitmaps).')
+  (N'1', N'Infrastructure error.'),
+  (N'2', N'Internal data integration process error or system generated event.'),
+  (N'3', N'Custom exception handling that has been implemented in code (Error Bitmaps).')
 
 MERGE [omd_metadata].[EVENT_TYPE] AS TARGET
 USING @tblMerge AS src
-    ON  TARGET.[EVENT_TYPE_CODE] = src.[EVENT_TYPE_CODE]
+  ON  TARGET.[EVENT_TYPE_CODE] = src.[EVENT_TYPE_CODE]
 
 WHEN MATCHED THEN
-    UPDATE
-    SET    [EVENT_TYPE_CODE_DESCRIPTION] = src.[EVENT_TYPE_CODE_DESCRIPTION]
+  UPDATE
+  SET      [EVENT_TYPE_CODE_DESCRIPTION] = src.[EVENT_TYPE_CODE_DESCRIPTION]
 
 WHEN NOT MATCHED THEN
-    INSERT  ([EVENT_TYPE_CODE]
-            ,[EVENT_TYPE_CODE_DESCRIPTION])
-    VALUES  ([EVENT_TYPE_CODE]
-            ,[EVENT_TYPE_CODE_DESCRIPTION]);
+  INSERT  ([EVENT_TYPE_CODE]
+          ,[EVENT_TYPE_CODE_DESCRIPTION])
+  VALUES  ([EVENT_TYPE_CODE]
+          ,[EVENT_TYPE_CODE_DESCRIPTION]);
 
 GO

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.EXECUTION_STATUS.sql

@@ -2,28 +2,45 @@
  * https://github.com/data-solution-automation-engine/DIRECT
  * Reference data insert and update script
  * DIRECT Framework v2.0
+
+ * Reference metadata table EXECUTION_STATUS stores execution status codes and descriptions.
+ * This script is used to insert and update reference data on deployment.
+ * Any bespoke execution status codes added manually to the target will be retained,
+ * as long as the keys differ.
+ * To maintain a clean CI/CD process, consider using this script to manage
+ * all reference data for execution status codes.
  *
  * [omd_metadata].[EXECUTION_STATUS]
  *
  ******************************************************************************/
 
 SET NOCOUNT ON;
 
-INSERT INTO [omd_metadata].[EXECUTION_STATUS]
-SELECT *
-FROM (
- /* EXECUTION_STATUS_CODE, EXECUTION_STATUS_DESCRIPTION */
- VALUES
+DECLARE @tblMerge TABLE(
+  [EXECUTION_STATUS_CODE]             NVARCHAR (100)  NOT NULL PRIMARY KEY CLUSTERED,
+  [EXECUTION_STATUS_DESCRIPTION]      NVARCHAR (4000) NULL
+);
+
+INSERT INTO @tblMerge([EXECUTION_STATUS_CODE], [EXECUTION_STATUS_DESCRIPTION])
+VALUES
   (N'Aborted',    N'An abort is an attempted execution which led to the instance unable to start. Abort means that the process did not run, but was supposed to. This is typically the result of incorrect configuration or race conditions in the orchestration. The most common reasons for an abort is that another instance of the same Batch or Module is already running. The same logical unit of processing can never run more than once at the same time to maintain data consistency. If this situation is detected the second process will abort before any data is processed. The Module (Instance) was executed from a parent Batch (Instance) but not registered as such in the Batch/Module relationship.'),
   (N'Cancelled',  N'The cancelled (skipped) status code indicates that the instance was attempted to be executed, but that the control framework found that it was not necessary to run the process. This can be due to Modules or Batches being disabled in the framework using the Active Indicator. Disabling processes can be done at Batch, Batch/Module and Module level. Another common scenario is that, when Batches are restarted, earlier successful Modules in that Batch will not be reprocessed. These Module Instances will be skipped / cancelled until the full Batch has completed successfully. This is to prevents data loss.'),
   (N'Executing',  N'The instance is currently running (executing). This is a transient state only, for when the process is actually running. As soon as it is completed this code will be updated to one of the end-state execution codes.'),
   (N'Failed',     N'The instance is no longer running after completing with failures.'),
   (N'Succeeded',  N'The instance is no longer running after successful completion of the process.')
- ) AS refData(EXECUTION_STATUS_CODE, EXECUTION_STATUS_DESCRIPTION)
-WHERE NOT EXISTS (
-  SELECT NULL
-  FROM [omd_metadata].[EXECUTION_STATUS] es
-  WHERE es.EXECUTION_STATUS_CODE = refData.EXECUTION_STATUS_CODE
-);
+
+MERGE [omd_metadata].[EXECUTION_STATUS] AS TARGET
+USING @tblMerge AS src
+  ON  TARGET.[EXECUTION_STATUS_CODE] = src.[EXECUTION_STATUS_CODE]
+
+WHEN MATCHED THEN
+  UPDATE
+  SET      [EXECUTION_STATUS_DESCRIPTION] = src.[EXECUTION_STATUS_DESCRIPTION]
+
+WHEN NOT MATCHED THEN
+  INSERT  ([EXECUTION_STATUS_CODE]
+          ,[EXECUTION_STATUS_DESCRIPTION])
+  VALUES  ([EXECUTION_STATUS_CODE]
+          ,[EXECUTION_STATUS_DESCRIPTION]);
 
 GO

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.FRAMEWORK_METADATA.sql

@@ -3,29 +3,52 @@
  * Reference data insert and update script
  * DIRECT Framework v2.0
  *
+ * Reference metadata table FRAMEWORK_METADATA stores metadata information.
+ * This script is used to insert and update reference data on deployment.
+ * Any bespoke metadata added manually to the target will be retained,
+ * as long as the keys differ.
+ * To maintain a clean CI/CD process, consider using this script to manage
+ * all reference data for metadata.
+ *
  * [omd_metadata].[FRAMEWORK_METADATA]
  *
  ******************************************************************************/
 
 SET NOCOUNT ON;
 
-INSERT INTO [omd_metadata].[FRAMEWORK_METADATA]
-SELECT *
-FROM (
-  /* [CODE], [VALUE], [GROUP], [DESCRIPTION], [ACTIVE_INDICATOR] */
-  VALUES
-  (
-    N'DIRECT_VERSION', N'2.0.0.0', N'SYSTEM_METADATA',
-    N'The current version of the DIRECT Framework and database', 'Y'
-  )
-) AS refData
-(
-  [CODE], [VALUE], [GROUP], [DESCRIPTION], [ACTIVE_INDICATOR]
-)
-WHERE NOT EXISTS (
-  SELECT NULL
-  FROM [omd_metadata].[FRAMEWORK_METADATA] m
-  WHERE m.[CODE] = refData.[CODE]
+DECLARE @tblMerge TABLE(
+  [CODE]             NVARCHAR (100)  NOT NULL PRIMARY KEY CLUSTERED,
+  [VALUE]            NVARCHAR (4000) NULL,
+  [GROUP]            NVARCHAR (100)  NOT NULL,
+  [DESCRIPTION]      NVARCHAR (4000) NULL,
+  [ACTIVE_INDICATOR] CHAR(1)         NOT NULL
 );
 
+INSERT INTO @tblMerge([CODE], [VALUE], [GROUP], [DESCRIPTION], [ACTIVE_INDICATOR])
+VALUES
+  (N'DIRECT_VERSION', N'2.0.0.0', N'SYSTEM_METADATA', N'The current version of the DIRECT Framework and database', 'Y')
+
+MERGE [omd_metadata].[FRAMEWORK_METADATA] AS TARGET
+USING @tblMerge AS src
+  ON  TARGET.[CODE] = src.[CODE]
+
+WHEN MATCHED THEN
+  UPDATE
+  SET      [VALUE] = src.[VALUE],
+           [GROUP] = src.[GROUP],
+           [DESCRIPTION] = src.[DESCRIPTION],
+           [ACTIVE_INDICATOR] = src.[ACTIVE_INDICATOR]
+
+WHEN NOT MATCHED THEN
+  INSERT  ([CODE]
+          ,[VALUE]
+          ,[GROUP]
+          ,[DESCRIPTION]
+          ,[ACTIVE_INDICATOR])
+  VALUES  ([CODE]
+          ,[VALUE]
+          ,[GROUP]
+          ,[DESCRIPTION]
+          ,[ACTIVE_INDICATOR]);
+
 GO

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.FREQUENCY.sql

@@ -3,8 +3,13 @@
  * Reference data insert and update script
  * DIRECT Framework v2.0
  *
- * Reference metadata table FREQUENCY
- * TODO: Add description
+ * Reference metadata table FREQUENCY stores frequency information.
+ * This table is used to define the frequency of the processing unit.
+ * This script is used to insert and update reference data on deployment.
+ * Any bespoke frequency added manually to the target will be retained,
+ * as long as the keys differ.
+ * To maintain a clean CI/CD process, consider using this script to manage
+ * all reference data for frequency.
  *
  * [omd_metadata].[FREQUENCY]
  *
@@ -19,7 +24,7 @@ DECLARE @tblMerge TABLE(
 
 INSERT INTO @tblMerge([FREQUENCY_CODE], [FREQUENCY_DESCRIPTION])
 VALUES
-  (N'Continuous', N'Continuously running integration processing.'),
+  (N'Continuous', N'Continuously running integration processing unit.'),
   (N'Triggered',  N'Event triggered processing unit.'),
   (N'On-demand',  N'On-demand processing unit.'),
   (N'Scheduled',  N'Scheduled processing unit.')

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.INTERNAL_PROCESSING_STATUS.sql

@@ -3,26 +3,42 @@
  * Reference data insert and update script
  * DIRECT Framework v2.0
  *
+ * Reference metadata table INTERNAL_PROCESSING_STATUS stores internal processing status codes and descriptions.
+ *
+ * This script is used to insert and update reference data on deployment.
+ * Any bespoke internal processing status codes added manually to the target will be retained,
+ * as long as the keys differ.
+ * To maintain a clean CI/CD process, consider using this script to manage
+ * all reference data for internal processing status codes.
+ *
  * [omd_metadata].[INTERNAL_PROCESSING_STATUS]
  *
  ******************************************************************************/
 
 SET NOCOUNT ON;
 
-INSERT INTO [omd_metadata].[INTERNAL_PROCESSING_STATUS]
-SELECT *
-FROM (
- /* INTERNAL_PROCESSING_STATUS_CODE, INTERNAL_INTERNAL_PROCESSING_STATUS_DESCRIPTION */
- VALUES
- (N'Abort',     N'This exception case indicates that the instance in question was executed, but that another instance of the same Batch or Module is already running (see also the equivalent Execution Status Code for additional detail). This is one of the checks performed before the regular process (Module and/or Batch) can continue. If this situation occurs, all processing should stop; no data should be processed. The process will use the Internal Processing Status `Abort` to trigger the Module/Batch `Abort` event which sets the Execution Status Code to `Cancelled`, ending the process gracefully.'),
- (N'Cancel',    N'The instance evaluation has determined that it is not necessary to run this process (see also the equivalent Execution Status Code for additional detail). As with Abort, if the Internal Process Status code is `Cancel` then all further processing should stop after the Execution Status Code has also been updated to `Cancel`.'),
- (N'Proceed',   N'The instance can continue on to the next processing. This is the default internal processing value; each process step will evaluate the Internal Process Status code and continue only if it is set to `Proceed`. After the pre-processing has been completed the `Proceed` value is the flag that is required to initiate the main process.'),
- (N'Rollback',  N'The `Rollback` code is only temporarily set during rollback execution in the Module Evaluation event. This is essentially for debugging purposes. After the rollback is completed the Internal Processing Status will be set to `Proceed` again to enable the continuation of the process.')
- ) AS refData(INTERNAL_PROCESSING_STATUS_CODE, INTERNAL_PROCESSING_STATUS_DESCRIPTION)
-WHERE NOT EXISTS (
-  SELECT NULL
-  FROM [omd_metadata].[INTERNAL_PROCESSING_STATUS] pin
-  WHERE pin.INTERNAL_PROCESSING_STATUS_CODE = refData.INTERNAL_PROCESSING_STATUS_CODE
-  );
+DECLARE @tblMerge TABLE(
+  [INTERNAL_PROCESSING_STATUS_CODE]             NVARCHAR (100)  NOT NULL PRIMARY KEY CLUSTERED,
+  [INTERNAL_PROCESSING_STATUS_DESCRIPTION]      NVARCHAR (4000) NULL
+);
+
+INSERT INTO @tblMerge([INTERNAL_PROCESSING_STATUS_CODE], [INTERNAL_PROCESSING_STATUS_DESCRIPTION])
+VALUES
+  (N'Abort',     N'This exception case indicates that the instance in question was executed, but that another instance of the same Batch or Module is already running (see also the equivalent Execution Status Code for additional detail). This is one of the checks performed before the regular process (Module and/or Batch) can continue. If this situation occurs, all processing should stop; no data should be processed. The process will use the Internal Processing Status `Abort` to trigger the Module/Batch `Abort` event which sets the Execution Status Code to `Cancelled`, ending the process gracefully.'),
+  (N'Cancel',    N'The instance evaluation has determined that it is not necessary to run this process (see also the equivalent Execution Status Code for additional detail). As with Abort, if the Internal Process Status code is `Cancel` then all further processing should stop after the Execution Status Code has also been updated to `Cancel`.'),
+  (N'Proceed',   N'The instance can continue on to the next processing. This is the default internal processing value; each process step will evaluate the Internal Process Status code and continue only if it is set to `Proceed`. After the pre-processing has been completed the `Proceed` value is the flag that is required to initiate the main process.'),
+  (N'Rollback',  N'The `Rollback` code is only temporarily set during rollback execution in the Module Evaluation event. This is essentially for debugging purposes. After the rollback is completed the Internal Processing Status will be set to `Proceed` again to enable the continuation of the process.');
+
+MERGE [omd_metadata].[INTERNAL_PROCESSING_STATUS] AS TARGET
+USING @tblMerge AS src
+  ON  TARGET.[INTERNAL_PROCESSING_STATUS_CODE] = src.[INTERNAL_PROCESSING_STATUS_CODE]
+WHEN MATCHED THEN
+  UPDATE
+  SET      [INTERNAL_PROCESSING_STATUS_DESCRIPTION] = src.[INTERNAL_PROCESSING_STATUS_DESCRIPTION]
+WHEN NOT MATCHED THEN
+  INSERT  ([INTERNAL_PROCESSING_STATUS_CODE]
+          ,[INTERNAL_PROCESSING_STATUS_DESCRIPTION])
+  VALUES  ([INTERNAL_PROCESSING_STATUS_CODE]
+          ,[INTERNAL_PROCESSING_STATUS_DESCRIPTION]);
 
 GO

Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.LAYER.sql

@@ -3,27 +3,42 @@
  * Reference data insert and update script
  * DIRECT Framework v2.0
  *
+ * Reference metadata table LAYER stores layer information.
+ *
+ * This script is used to insert and update reference data on deployment.
+ * Any bespoke layers added manually to the target will be retained,
+ * as long as the keys differ.
+ * To maintain a clean CI/CD process, consider using this script to manage
+ * all reference data for layers.
+ *
  * [omd_metadata].[LAYER]
  *
  ******************************************************************************/
 
 SET NOCOUNT ON;
 
-INSERT INTO [omd_metadata].[LAYER]
-SELECT *
-FROM
-(
- /* LAYER_CODE, in  */
-  VALUES
-    (N'Integration',    N'The Integration Layer'),
-    (N'Presentation',   N'The Presentation Layer'),
-    (N'Staging',        N'The Staging Layer'),
-    (N'Maintenance',    N'Internal Data Solution')
-) AS refData(LAYER_CODE, LAYER_DESCRIPTION)
-WHERE NOT EXISTS (
-  SELECT NULL
-  FROM [omd_metadata].[LAYER] l
-  WHERE l.LAYER_CODE = refData.LAYER_CODE
+DECLARE @tblMerge TABLE(
+  [LAYER_CODE]             NVARCHAR (100)  NOT NULL PRIMARY KEY CLUSTERED,
+  [LAYER_DESCRIPTION]      NVARCHAR (4000) NULL
 );
 
+INSERT INTO @tblMerge([LAYER_CODE], [LAYER_DESCRIPTION])
+VALUES
+  (N'Integration',    N'The Integration Layer'),
+  (N'Presentation',   N'The Presentation Layer'),
+  (N'Staging',        N'The Staging Layer'),
+  (N'Maintenance',    N'Internal Data Solution')
+
+MERGE [omd_metadata].[LAYER] AS TARGET
+USING @tblMerge AS src
+  ON  TARGET.[LAYER_CODE] = src.[LAYER_CODE]
+WHEN MATCHED THEN
+  UPDATE
+  SET      [LAYER_DESCRIPTION] = src.[LAYER_DESCRIPTION]
+WHEN NOT MATCHED THEN
+  INSERT  ([LAYER_CODE]
+          ,[LAYER_DESCRIPTION])
+  VALUES  ([LAYER_CODE]
+          ,[LAYER_DESCRIPTION]);
+
 GO