Skip to content

Commit 2e67ede

Browse files
RoelantVosRoelantVos
committed
Updated to release 1 for DIRECT, all cleaned up.
1 parent af3cb9a commit 2e67ede

File tree

69 files changed

+1710
-1084
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+1710
-1084
lines changed

‎020_ETL_Solution_For_SSIS/Configuration_SQL_Server.txt‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
connectionStringSTG|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
2-
connectionStringHSTG|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
3-
connectionStringDIRECT|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
4-
connectionStringGEN|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
5-
connectionStringINT|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
6-
connectionStringPRES|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
7-
connectionStringSOURCE|Provider=SQLNCLI11;Server=AUBRIDEBIW02;Initial Catalog=PDV_RV2;Integrated Security=SSPI;
1+
connectionStringSTG|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
2+
connectionStringHSTG|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
3+
connectionStringDIRECT|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
4+
connectionStringGEN|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
5+
connectionStringINT|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
6+
connectionStringPRES|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
7+
connectionStringSOURCE|Provider=SQLNCLI11;Server=BISERVER;Initial Catalog=EDW_900_OMD_Framework;User Id=sa;Password=K3kobus2;
88
STGPackagePrefix|m_100_STG
99
SourceSystemPrefix|THEDEFAULTISUSED
1010
SourceSystemPrefixEXAMPLE|EXAMPLE

‎020_ETL_Solution_For_SSIS/DIRECT_ETL_Solution_For_SSIS.sln‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "References", "References",
1717
000_References\System.Windows.Forms.dll = 000_References\System.Windows.Forms.dll
1818
EndProjectSection
1919
EndProject
20+
Project("{159641D6-6404-4A2A-AE62-294DE0FE8301}") = "Shared", "Shared\Shared.dtproj", "{09341913-6A41-4AB8-A826-FFEA05B80E17}"
21+
EndProject
2022
Global
2123
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2224
Debug|Any CPU = Debug|Any CPU
@@ -33,6 +35,14 @@ Global
3335
{0CF7F3E3-9E5D-4769-9284-D91DDD7FD864}.Production|Any CPU.Build.0 = Production
3436
{0CF7F3E3-9E5D-4769-9284-D91DDD7FD864}.Release|Any CPU.ActiveCfg = Production
3537
{0CF7F3E3-9E5D-4769-9284-D91DDD7FD864}.Release|Any CPU.Build.0 = Production
38+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Debug|Any CPU.ActiveCfg = Development
39+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Debug|Any CPU.Build.0 = Development
40+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Development|Any CPU.ActiveCfg = Development
41+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Development|Any CPU.Build.0 = Development
42+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Production|Any CPU.ActiveCfg = Development
43+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Production|Any CPU.Build.0 = Development
44+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Release|Any CPU.ActiveCfg = Development
45+
{09341913-6A41-4AB8-A826-FFEA05B80E17}.Release|Any CPU.Build.0 = Development
3646
EndGlobalSection
3747
GlobalSection(SolutionProperties) = preSolution
3848
HideSolutionNode = FALSE

‎020_ETL_Solution_For_SSIS/Enterprise_Data_Warehouse/000_Batch_Empty_Wrapper.biml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
<!--
2+
Purpose: Base generation template for the Batch package.
3+
Comments: Uses file storage, fully functional.
4+
Last update: Added include files for event handling
5+
-->
6+
7+
<#@ include file="../Shared/Include_ProjectConfiguration.biml" #>
8+
<#@ include file="../Shared/Include_C_References.biml" #>
9+
<#@ include file="../Shared/Include_BIML_Configuration.biml" #>
10+
11+
12+
<#
13+
string sqlStatementForTablesToImport = @"SELECT 'EXAMPLE' AS [TABLE_NAME]";
14+
15+
string currentTableName;
16+
string SourceTableName;
17+
string HistTableName;
18+
#>
19+
20+
<Biml xmlns="http://schemas.varigence.com/biml.xsd">
21+
22+
<ScriptProjects>
23+
<#@ include file="../Shared/Include_Generic_ScriptProject.biml" #>
24+
</ScriptProjects>
25+
26+
<Connections>
27+
<Connection Name="DIRECT" CreateInProject="true" ConnectionString="<#=connectionStringDIRECT#>"/>
28+
</Connections>
29+
30+
<Packages>
31+
<#
32+
DataTable tables = ExternalDataAccess.GetDataTable(connectionStringSTG, sqlStatementForTablesToImport);
33+
34+
foreach (DataRow row in tables.Rows)
35+
{
36+
SourceTableName = (string)row["TABLE_NAME"];
37+
HistTableName = "H"+SourceTableName;
38+
#>
39+
40+
<Package Name="b_EDW_STG_INT_<#=SourceTableName#>" Language="None" LoggingMode="Disabled" ConstraintMode="Parallel">
41+
42+
<#@ include file="../Shared/Include_Variables_Batch.biml" #>
43+
<#@ include file="../Shared/Include_Batch_OnError_Event.biml" #>
44+
45+
<Tasks>
46+
47+
<#@ include file="../Shared/Include_Batch_OnPreExecute_Event.biml" #>
48+
49+
<Container Name="Batch Processing" ConstraintMode="Parallel">
50+
<PrecedenceConstraints LogicalType="Or">
51+
<Inputs>
52+
<Input OutputPathName="SCT - Dummy Placeholder.Output" EvaluationOperation="ExpressionAndConstraint" Expression="@V_PROCESSING_INDICATOR == &quot;P&quot;"/>
53+
</Inputs>
54+
</PrecedenceConstraints>
55+
56+
<Tasks>
57+
58+
<ExecutePackage Name="EPT - <#=SourceTableName#>">
59+
<ExternalProjectPackage Package="<#=SourceTableName#>.dtsx" />
60+
<ParameterBindings>
61+
<ParameterBinding Name="V_BATCH_ID" VariableName="User.V_BATCH_ID" />
62+
<ParameterBinding Name="V_BATCH_INSTANCE_ID" VariableName="User.V_BATCH_INSTANCE_ID" />
63+
<ParameterBinding Name="V_BATCH_INSTANCE_START_DATETIME" VariableName="User.V_BATCH_INSTANCE_START_DATETIME" />
64+
</ParameterBindings>
65+
<Annotations>
66+
<Annotation AnnotationType="Description">Staging Area processing.</Annotation>
67+
</Annotations>
68+
</ExecutePackage>
69+
70+
<ExecutePackage Name="EPT - <#=HistTableName#>">
71+
<ExternalProjectPackage Package="<#=HistTableName#>.dtsx" />
72+
73+
<PrecedenceConstraints>
74+
<Inputs>
75+
<Input OutputPathName="EPT - <#=SourceTableName#>.Output" EvaluationOperation="Constraint" />
76+
</Inputs>
77+
</PrecedenceConstraints>
78+
79+
<ParameterBindings>
80+
<ParameterBinding Name="V_BATCH_ID" VariableName="User.V_BATCH_ID" />
81+
<ParameterBinding Name="V_BATCH_INSTANCE_ID" VariableName="User.V_BATCH_INSTANCE_ID" />
82+
<ParameterBinding Name="V_BATCH_INSTANCE_START_DATETIME" VariableName="User.V_BATCH_INSTANCE_START_DATETIME" />
83+
</ParameterBindings>
84+
<Annotations>
85+
<Annotation AnnotationType="Description">Persistent Staging Area processing.</Annotation>
86+
</Annotations>
87+
</ExecutePackage>
88+
89+
</Tasks>
90+
91+
</Container>
92+
93+
<#@ include file="../Shared/Include_Batch_OnPostExecute_Event.biml" #>
94+
95+
</Tasks>
96+
97+
</Package>
98+
99+
<#
100+
}
101+
#>
102+
103+
</Packages>
104+
</Biml>

‎020_ETL_Solution_For_SSIS/Enterprise_Data_Warehouse/000_m_100_Empty_Wrapper.biml‎ renamed to ‎020_ETL_Solution_For_SSIS/Enterprise_Data_Warehouse/000_Module_Empty_Wrapper.biml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
Comments:
88
-->
99

10-
<#@ include file="Include_ProjectConfiguration.biml" #>
11-
<#@ include file="Include_C_References.biml" #>
12-
<#@ include file="Include_BIML_Configuration.biml" #>
10+
<#@ include file="../Shared/Include_ProjectConfiguration.biml" #>
11+
<#@ include file="../Shared/Include_C_References.biml" #>
12+
<#@ include file="../Shared/Include_BIML_Configuration.biml" #>
1313

1414
<#
1515
// This parameter is meant to identify the Biml pattern, so it can only be applied to source-to-staging ETLs that are administered to use it.
@@ -29,10 +29,10 @@
2929
<Biml>
3030

3131
<ScriptProjects>
32-
<#@ include file="Include_STG_ScriptProject.biml" #>
32+
<#@ include file="../Shared/Include_STG_ScriptProject.biml" #>
3333
</ScriptProjects>
3434

35-
<#@ include file="Include_STG_Connections.biml" #>
35+
<#@ include file="../Shared/Include_STG_Connections.biml" #>
3636

3737
<Packages>
3838

@@ -62,13 +62,13 @@
6262
<Annotation AnnotationType="Description">Package <#=PackagePrefix#>_<#=SourceSystemPrefix#>_<#=SourceTableName.ToUpper()#> generated at <#=currentDateTime#>.</Annotation>
6363
</Annotations>
6464

65-
<#@ include file="Include_Parameters.biml" #>
66-
<#@ include file="Include_Variables_STG.biml" #>
67-
<#@ include file="Include_OnError_Event.biml" #>
65+
<#@ include file="../Shared/Include_Parameters.biml" #>
66+
<#@ include file="../Shared/Include_Variables_Module.biml" #>
67+
<#@ include file="../Shared/Include_OnError_Event.biml" #>
6868

6969
<Tasks>
7070

71-
<#@ include file="Include_OnPreExecute_Event.biml" #>
71+
<#@ include file="../Shared/Include_OnPreExecute_Event.biml" #>
7272

7373
<Dataflow Name="DFT - <#=StagingAreaTableName#>" DelayValidation="true" FailPackageOnFailure="true">
7474

@@ -78,7 +78,7 @@
7878
</Inputs>
7979
</PrecedenceConstraints>
8080

81-
<#@ include file="Include_OnPostExecute_Event.biml" #>
81+
<#@ include file="../Shared/Include_OnPostExecute_Event.biml" #>
8282

8383

8484
<Annotations>

‎020_ETL_Solution_For_SSIS/Enterprise_Data_Warehouse/000_b_Batch_Empty_Wrapper.biml‎

Lines changed: 0 additions & 100 deletions
This file was deleted.

‎020_ETL_Solution_For_SSIS/Enterprise_Data_Warehouse/Batch_Abort.dtsx‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
<?xml version="1.0" encoding="utf-8"?>
2-
<DTS:Executable DTS:CreationDate="06/10/2019 17:48:56" DTS:CreationName="Microsoft.Package" DTS:CreatorComputerName="LAUBRI006619" DTS:CreatorName="rvos" DTS:Description="Update the information to record the abort of a Batch Instance. This is done by:&#xA; - Setting the Execution Status Code to 'A'&#xA; - Setting the Next_Run_Flag to 'E'&#xA; - Setting the End_Datetime for the package run to the system datetime (now)" DTS:DTSID="{89DDE970-B4DD-4879-AF45-BCC8D62CCFD2}" DTS:EnableConfig="True" DTS:ExecutableType="Microsoft.Package" DTS:LastModifiedProductVersion="" DTS:LocaleID="3081" DTS:ObjectName="Batch_Abort" DTS:ProtectionLevel="0" DTS:refId="Package" DTS:VersionBuild="0" DTS:VersionGUID="{7E51E714-B2A6-4E98-9132-4A744A55A154}" DTS:VersionMajor="0" xmlns:DTS="www.microsoft.com/SqlServer/Dts">
2+
<DTS:Executable DTS:CreationDate="06/12/2019 13:01:15" DTS:CreationName="Microsoft.Package" DTS:CreatorComputerName="LAPTOP-RBS8FBTA" DTS:CreatorName="roela" DTS:Description="Update the information to record the abort of a Batch Instance. This is done by:&#xA; - Setting the Execution Status Code to 'A'&#xA; - Setting the Next_Run_Flag to 'E'&#xA; - Setting the End_Datetime for the package run to the system datetime (now)" DTS:DTSID="{4CA9AABE-55FB-48D1-8ED8-5C71ABE81464}" DTS:EnableConfig="True" DTS:ExecutableType="Microsoft.Package" DTS:LastModifiedProductVersion="" DTS:LocaleID="3081" DTS:ObjectName="Batch_Abort" DTS:ProtectionLevel="0" DTS:refId="Package" DTS:VersionBuild="0" DTS:VersionGUID="{160AB071-17A7-4C7C-B859-955ED79685B0}" DTS:VersionMajor="0" xmlns:DTS="www.microsoft.com/SqlServer/Dts">
33
<DTS:Property DTS:Name="PackageFormatVersion">8</DTS:Property>
44
<DTS:PackageParameters>
5-
<DTS:PackageParameter DTS:CreationName="" DTS:DataType="3" DTS:Description="" DTS:DTSID="{B5EFAA12-FB64-4A75-81F9-4C280B141282}" DTS:ObjectName="V_BATCH_INSTANCE_ID" DTS:Required="0" DTS:Sensitive="0">
5+
<DTS:PackageParameter DTS:CreationName="" DTS:DataType="3" DTS:Description="" DTS:DTSID="{055B7784-2AF9-4AD9-9DC2-9057BE35367B}" DTS:ObjectName="V_BATCH_INSTANCE_ID" DTS:Required="0" DTS:Sensitive="0">
66
<DTS:Property DTS:DataType="3" DTS:Name="ParameterValue">0</DTS:Property>
77
</DTS:PackageParameter>
88
</DTS:PackageParameters>
99
<DTS:Variables>
10-
<DTS:Variable DTS:CreationName="" DTS:DTSID="{8CBFEB0E-2BF2-4378-89DB-98EF66CA6745}" DTS:EvaluateAsExpression="True" DTS:Expression="@[$Package::V_BATCH_INSTANCE_ID]" DTS:IncludeInDebugDump="6789" DTS:Namespace="User" DTS:ObjectName="V_BATCH_INSTANCE_ID">
10+
<DTS:Variable DTS:CreationName="" DTS:DTSID="{DA3126A7-39B0-441F-9CD8-BB39060738A2}" DTS:EvaluateAsExpression="True" DTS:Expression="@[$Package::V_BATCH_INSTANCE_ID]" DTS:IncludeInDebugDump="6789" DTS:Namespace="User" DTS:ObjectName="V_BATCH_INSTANCE_ID">
1111
<DTS:VariableValue DTS:DataType="3">0</DTS:VariableValue>
1212
</DTS:Variable>
1313
</DTS:Variables>
1414
<DTS:Executables>
15-
<DTS:Executable DTS:CreationName="STOCK:SEQUENCE" DTS:Description="Batch Abort" DTS:DTSID="{BE845C7F-EA85-47F4-942B-DBFFA3E64135}" DTS:ExecutableType="STOCK:SEQUENCE" DTS:LocaleID="-1" DTS:ObjectName="Batch Abort" DTS:refId="Package\Batch Abort">
15+
<DTS:Executable DTS:CreationName="STOCK:SEQUENCE" DTS:Description="Batch Abort" DTS:DTSID="{AB4B3D32-5543-4D80-94D2-8070A23E79E0}" DTS:ExecutableType="STOCK:SEQUENCE" DTS:LocaleID="-1" DTS:ObjectName="Batch Abort" DTS:refId="Package\Batch Abort">
1616
<DTS:Executables>
17-
<DTS:Executable DTS:CreationName="Microsoft.ExecuteSQLTask" DTS:Description="Execute SQL Task" DTS:DTSID="{1B10BB5C-1C40-448B-B3DA-68D4C9648503}" DTS:ExecutableType="Microsoft.ExecuteSQLTask" DTS:LocaleID="-1" DTS:ObjectName="EST - Update the Batch Instance for Abort" DTS:refId="Package\Batch Abort\EST - Update the Batch Instance for Abort" DTS:TaskContact="Execute SQL Task; Microsoft Corporation; SQL Server 2014; © 2014 Microsoft Corporation; All Rights Reserved;http://www.microsoft.com/sql/support/default.asp;1">
17+
<DTS:Executable DTS:CreationName="Microsoft.ExecuteSQLTask" DTS:Description="Execute SQL Task" DTS:DTSID="{7184A680-1582-42A1-9673-F7D265057989}" DTS:ExecutableType="Microsoft.ExecuteSQLTask" DTS:LocaleID="-1" DTS:ObjectName="EST - Update the Batch Instance for Abort" DTS:refId="Package\Batch Abort\EST - Update the Batch Instance for Abort" DTS:TaskContact="Execute SQL Task; Microsoft Corporation; SQL Server 2014; © 2014 Microsoft Corporation; All Rights Reserved;http://www.microsoft.com/sql/support/default.asp;1">
1818
<DTS:ObjectData>
1919
<SQLTask:SqlTaskData SQLTask:Connection="DIRECT" SQLTask:SqlStatementSource="UPDATE BATCH_INSTANCE SET EXECUTION_STATUS_CODE='A', NEXT_RUN_INDICATOR = 'P', END_DATETIME=GETDATE() WHERE BATCH_INSTANCE_ID=?" xmlns:SQLTask="www.microsoft.com/sqlserver/dts/tasks/sqltask">
2020
<SQLTask:ParameterBinding SQLTask:DataType="3" SQLTask:DtsVariableName="User::V_BATCH_INSTANCE_ID" SQLTask:ParameterDirection="Input" SQLTask:ParameterName="0" SQLTask:ParameterSize="-1" />
@@ -25,6 +25,6 @@
2525
</DTS:Executable>
2626
</DTS:Executables>
2727
<DTS:Configurations>
28-
<DTS:Configuration DTS:ConfigurationString="V_BATCH_INSTANCE_ID" DTS:ConfigurationVariable="\Package.Variables[User::V_BATCH_INSTANCE_ID].Properties[Value]" DTS:CreationName="" DTS:DTSID="{7AB22D17-18DE-430F-A66D-441127429AD4}" DTS:ObjectName="V_BATCH_INSTANCE_ID" />
28+
<DTS:Configuration DTS:ConfigurationString="V_BATCH_INSTANCE_ID" DTS:ConfigurationVariable="\Package.Variables[User::V_BATCH_INSTANCE_ID].Properties[Value]" DTS:CreationName="" DTS:DTSID="{7F7B9E2E-578B-463A-927D-48CC30E735DF}" DTS:ObjectName="V_BATCH_INSTANCE_ID" />
2929
</DTS:Configurations>
3030
</DTS:Executable>

0 commit comments

Comments
 (0)