Skip to content

Commit 6872eb7

Browse files
whitneygriffithwhitneygriffith
committed
add staging tables
1 parent 0ebb286 commit 6872eb7

47 files changed

Lines changed: 339 additions & 211 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎sql/SPROC_move_data_to_permanent_tables.sql‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,27 @@ CREATE PROCEDURE move_data_to_permanent_tables @watermark_table NVARCHAR(100)
22
AS
33
-- SET XACT_ABORT ON will cause the transaction to be uncommittable
44
-- when the constraint violation occurs.
5-
SET XACT_ABORT ON;
5+
SET XACT_ABORT ON;
6+
BEGIN TRANSACTION move_data_to_permanent_tables
67
BEGIN TRY
7-
BEGIN TRANSACTION move_data_to_permanent_tables
8-
98
DECLARE @current_table_name VARCHAR(MAX);
109
DECLARE @staging_table_name VARCHAR(MAX);
1110
DECLARE @log_date AS NVARCHAR(50);
1211
DECLARE @log_message NVARCHAR(512);
1312
SET @current_table_name = ''
1413

14+
-- The use of RAISERROR with the NOWAIT option shouldn't actually be emitting an exception as you might expect. It's used
15+
-- to immediately emmit log/console messages. For additional details, see
16+
-- https://www.mssqltips.com/sqlservertip/1660/using-the-nowait-option-with-the-sql-server-raiserror-statement/
17+
1518
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
1619
SET @log_message = @log_date + ' move_data_to_permanent_tables is starting execution'
1720
RAISERROR (@log_message, 0, 1) WITH NOWAIT
1821

1922
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
2023
SET @log_message = @log_date + ' move_data_to_permanent_tables is removing all indexes and constraints on OMOP tables'
2124
RAISERROR (@log_message, 0, 1) WITH NOWAIT
22-
25+
2326
EXEC dbo.remove_indexes_constraints;
2427
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
2528
SET @log_message = @log_date + ' move_data_to_permanent_tables removed all indexes and constraints on OMOP tables'
@@ -28,7 +31,7 @@ AS
2831
DECLARE @cursor_sql VARCHAR(MAX)
2932
SET @cursor_sql = 'DECLARE omop_tables_cursor CURSOR FOR SELECT table_name FROM ' + @watermark_table
3033
EXEC(@cursor_sql)
31-
34+
3235
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
3336
SET @log_message = @log_date + ' move_data_to_permanent_tables is moving data for tables in ' + @watermark_table
3437
RAISERROR (@log_message, 0, 1) WITH NOWAIT
@@ -82,24 +85,27 @@ AS
8285

8386
IF (SELECT CURSOR_STATUS('global','omop_tables_cursor')) >= -1
8487
BEGIN
85-
DEALLOCATE omop_tables_cursor;
88+
DEALLOCATE omop_tables_cursor;
8689
END
8790

8891
-- Transaction uncommittable
8992
IF (XACT_STATE()) = -1
90-
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
91-
SET @log_message = @log_date + ' move_data_to_permanent_tables is rolling back transaction'
92-
RAISERROR (@log_message, 0, 1) WITH NOWAIT
93-
ROLLBACK TRANSACTION move_data_to_permanent_tables;
94-
93+
BEGIN
94+
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
95+
SET @log_message = @log_date + ' move_data_to_permanent_tables is rolling back transaction'
96+
RAISERROR (@log_message, 0, 1) WITH NOWAIT
97+
ROLLBACK TRANSACTION move_data_to_permanent_tables;
98+
THROW;
99+
END
95100
-- Transaction committable
96101
IF (XACT_STATE()) = 1
97-
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
98-
SET @log_message = @log_date + ' move_data_to_permanent_tables is committing transaction'
99-
RAISERROR (@log_message, 0, 1) WITH NOWAIT
100-
COMMIT TRANSACTION move_data_to_permanent_tables;
101-
102-
THROW;
102+
BEGIN
103+
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
104+
SET @log_message = @log_date + ' move_data_to_permanent_tables is committing transaction'
105+
RAISERROR (@log_message, 0, 1) WITH NOWAIT
106+
COMMIT TRANSACTION move_data_to_permanent_tables;
107+
END
103108

104109
END CATCH;
105110

111+
GO

‎sql/SPROC_test_move_data_to_permanent_tables.sql‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,31 +29,42 @@ BEGIN
2929
EXEC dbo.move_data_to_permanent_tables 'watermark_test';
3030

3131

32+
DECLARE @Expected AS VARCHAR(100)
33+
DECLARE @Actual AS VARCHAR(100)
34+
35+
SET @Expected = (SELECT COUNT(*) FROM expected_test_table)
36+
SET @Actual = (SELECT COUNT(*) FROM test_table)
3237
-- Validate test table has the same number of rows as expected table
3338
IF (SELECT COUNT(*) FROM test_table) = (SELECT COUNT(*) FROM expected_test_table)
34-
RAISERROR ('Equal', 0, 1) WITH NOWAIT
35-
ELSE
36-
RAISERROR ('Not Equal', 0, 1);
39+
RAISERROR ('Expected and Actual Count is Equal', 0, 1) WITH NOWAIT;
40+
ELSE
41+
RAISERROR ('Actual Value(%s) is not Equal to expected value (%s) ', 0, 1, @Actual, @Expected) WITH NOWAIT;
3742

3843
-- Validate test table has the same value for col1 as expected table
44+
SET @Expected = (SELECT col1 FROM expected_test_table)
45+
SET @Actual = (SELECT col1 FROM test_table)
3946
IF (SELECT col1 FROM test_table) = (SELECT col1 FROM expected_test_table)
40-
RAISERROR ('Equal', 0, 1) WITH NOWAIT;
47+
RAISERROR ('Actual Col 1 is Equal to Expected Col 1', 0, 1) WITH NOWAIT;
4148
ELSE
42-
RAISERROR ('Not Equal', 0, 1);
49+
RAISERROR ('Actual Value(%s) is not Equal to expected value (%s) ', 0, 1, @Actual, @Expected) WITH NOWAIT;
4350

4451

4552
-- Validate test table has the same value for col2 as expected table
53+
SET @Expected = (SELECT col2 FROM expected_test_table)
54+
SET @Actual = (SELECT col2 FROM test_table)
4655
IF (SELECT col2 FROM test_table) = (SELECT col2 FROM expected_test_table)
47-
RAISERROR ('Equal', 0, 1) WITH NOWAIT
56+
RAISERROR ('Actual Col 2 is Equal to Expected Col 2', 0, 1) WITH NOWAIT;
4857
ELSE
49-
RAISERROR ('Not Equal', 0, 1);
58+
RAISERROR ('Actual Value(%s) is not Equal to expected value (%s) ', 0, 1, @Actual, @Expected) WITH NOWAIT;
5059

5160
-- Validate test table has the same value for col3 as expected table
61+
SET @Expected = (SELECT col3 FROM expected_test_table)
62+
SET @Actual = (SELECT col3 FROM test_table)
5263
IF (SELECT col3 FROM test_table) = (SELECT col3 FROM expected_test_table)
53-
RAISERROR ('Equal', 0, 1) WITH NOWAIT
64+
RAISERROR ('Actual Col 3 is Equal to Expected Col 3', 0, 1) WITH NOWAIT;
5465
ELSE
55-
RAISERROR ('Not Equal', 0, 1);
56-
66+
RAISERROR ('Actual Value(%s) is not Equal to expected value (%s) ', 0, 1, @Actual, @Expected) WITH NOWAIT;
67+
5768
DROP TABLE watermark_test;
5869
DROP TABLE test_table;
5970
DROP TABLE staging_test_table;

‎sql/dacpac/dbo/Scripts/Script.PostDeployment.sql‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎sql/dacpac/dbo/StoredProcedures/move_data_to_permanent_tables.sql‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,19 @@ CREATE PROCEDURE move_data_to_permanent_tables @watermark_table NVARCHAR(100)
22
AS
33
-- SET XACT_ABORT ON will cause the transaction to be uncommittable
44
-- when the constraint violation occurs.
5-
SET XACT_ABORT ON;
5+
SET XACT_ABORT ON;
6+
BEGIN TRANSACTION move_data_to_permanent_tables
67
BEGIN TRY
7-
BEGIN TRANSACTION move_data_to_permanent_tables
8-
98
DECLARE @current_table_name VARCHAR(MAX);
109
DECLARE @staging_table_name VARCHAR(MAX);
1110
DECLARE @log_date AS NVARCHAR(50);
1211
DECLARE @log_message NVARCHAR(512);
1312
SET @current_table_name = ''
1413

14+
-- The use of RAISERROR with the NOWAIT option shouldn't actually be emitting an exception as you might expect. It's used
15+
-- to immediately emmit log/console messages. For additional details, see
16+
-- https://www.mssqltips.com/sqlservertip/1660/using-the-nowait-option-with-the-sql-server-raiserror-statement/
17+
1518
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
1619
SET @log_message = @log_date + ' move_data_to_permanent_tables is starting execution'
1720
RAISERROR (@log_message, 0, 1) WITH NOWAIT
@@ -82,26 +85,27 @@ AS
8285

8386
IF (SELECT CURSOR_STATUS('global','omop_tables_cursor')) >= -1
8487
BEGIN
85-
DEALLOCATE omop_tables_cursor;
88+
DEALLOCATE omop_tables_cursor;
8689
END
8790

8891
-- Transaction uncommittable
8992
IF (XACT_STATE()) = -1
90-
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
91-
SET @log_message = @log_date + ' move_data_to_permanent_tables is rolling back transaction'
92-
RAISERROR (@log_message, 0, 1) WITH NOWAIT
93-
ROLLBACK TRANSACTION move_data_to_permanent_tables;
94-
93+
BEGIN
94+
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
95+
SET @log_message = @log_date + ' move_data_to_permanent_tables is rolling back transaction'
96+
RAISERROR (@log_message, 0, 1) WITH NOWAIT
97+
ROLLBACK TRANSACTION move_data_to_permanent_tables;
98+
THROW;
99+
END
95100
-- Transaction committable
96101
IF (XACT_STATE()) = 1
97-
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
98-
SET @log_message = @log_date + ' move_data_to_permanent_tables is committing transaction'
99-
RAISERROR (@log_message, 0, 1) WITH NOWAIT
100-
COMMIT TRANSACTION move_data_to_permanent_tables;
101-
102-
THROW;
102+
BEGIN
103+
SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
104+
SET @log_message = @log_date + ' move_data_to_permanent_tables is committing transaction'
105+
RAISERROR (@log_message, 0, 1) WITH NOWAIT
106+
COMMIT TRANSACTION move_data_to_permanent_tables;
107+
END
103108

104109
END CATCH;
105110

106111
GO
107-

‎sql/dacpac/dbo/StoredProcedures/test_move_data_to_permanent_tables.sql‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎sql/dacpac/dbo/Tables/metadata.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CREATE TABLE [dbo].[metadata] (
2+
[metadata_concept_id] INT NOT NULL,
3+
[metadata_type_concept_id] INT NOT NULL,
4+
[name] VARCHAR (250) NOT NULL,
5+
[value_as_string] VARCHAR (MAX) NULL,
6+
[value_as_concept_id] INT NULL,
7+
[metadata_date] DATE NULL,
8+
[metadata_datetime] DATETIME2 (7) NULL
9+
);

‎sql/dacpac/dbo/Tables/staging_attribute_definition.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
CREATE TABLE [dbo].[staging_attribute_definition] (
2+
[attribute_definition_id] INT NOT NULL,
3+
[attribute_name] VARCHAR (255) NOT NULL,
4+
[attribute_description] VARCHAR (MAX) NULL,
5+
[attribute_type_concept_id] INT NOT NULL,
6+
[attribute_syntax] VARCHAR (MAX) NULL,
7+
);
8+

‎sql/dacpac/dbo/Tables/staging_care_site.sql‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,3 @@ CREATE TABLE [dbo].[staging_care_site] (
66
[care_site_source_value] VARCHAR (50) NULL,
77
[place_of_service_source_value] VARCHAR (50) NULL
88
);
9-
10-
11-
GO
12-

‎sql/dacpac/dbo/Tables/staging_cdm_source.sql‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,3 @@ CREATE TABLE [dbo].[staging_cdm_source] (
1111
[vocabulary_version] VARCHAR (20) NULL
1212
);
1313

14-
15-
GO
16-

‎sql/dacpac/dbo/Tables/staging_cohort.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
CREATE TABLE [dbo].[staging_cohort] (
2+
[cohort_definition_id] INT NOT NULL,
3+
[subject_id] INT NOT NULL,
4+
[cohort_start_date] DATE NOT NULL,
5+
[cohort_end_date] DATE NOT NULL,
6+
);
7+
8+

0 commit comments

Comments
 (0)