Skip to content

Commit 3814571

Browse files
author
Wee Hyong Tok
committed
Updated ADF Walkthrough Samples
Update JSON files to the latest format.
1 parent 2a087d2 commit 3814571

17 files changed

+720
-473
lines changed

‎Samples/ADFWalkthrough/LinkedServices/AzureSqlLinkedService.json‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
2-
"name": "AzureSqlLinkedService",
3-
"properties":
4-
{
5-
"type": "AzureSqlLinkedService",
6-
"connectionString": "Server=tcp:<azure sql server>.database.windows.net,1433;Database=MarketingCampaigns;User ID=<user name>;Password=<password>;Trusted_Connection=False;Encrypt=True;Connection Timeout=30"
2+
"name": "AzureSqlLinkedService",
3+
"properties": {
4+
"type": "AzureSqlDatabase",
5+
"typeProperties": {
6+
"connectionString": "Server=tcp:<azure sql server>.database.windows.net,1433;Database=MarketingCampaigns;User ID=<user name>;Password=<password>;Trusted_Connection=False;Encrypt=True;Connection Timeout=30"
77
}
8-
}
8+
}
9+
}

‎Samples/ADFWalkthrough/LinkedServices/HDInsightLinkedService.json‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
2-
"name": "HDInsightLinkedService",
3-
"properties":
4-
{
5-
"type": "HDInsightOnDemandLinkedService",
6-
"clusterSize": "4",
7-
"jobsContainer": "adfjobs",
8-
"timeToLive": "00:05:00",
9-
"linkedServiceName": "HDInsightStorageLinkedService"
2+
"name": "HDInsightLinkedService",
3+
"properties": {
4+
"type": "HDInsightOnDemand",
5+
"typeProperties": {
6+
"clusterSize": "4",
7+
"jobsContainer": "adfjobs",
8+
"timeToLive": "00:05:00",
9+
"linkedServiceName": "HDInsightStorageLinkedService"
1010
}
11+
}
1112
}

‎Samples/ADFWalkthrough/LinkedServices/HDInsightStorageLinkedService.json‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
2-
"name": "HDInsightStorageLinkedService",
3-
"properties":
4-
{
5-
"type": "AzureStorageLinkedService",
6-
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<account key>"
2+
"name": "HDInsightStorageLinkedService",
3+
"properties": {
4+
"type": "AzureStorage",
5+
"typeProperties": {
6+
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<account key>"
77
}
8+
}
89
}

‎Samples/ADFWalkthrough/LinkedServices/StorageLinkedService.json‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
2-
"name": "StorageLinkedService",
3-
"properties":
4-
{
5-
"type": "AzureStorageLinkedService",
6-
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<Account key>"
2+
"name": "StorageLinkedService",
3+
"properties": {
4+
"type": "AzureStorage",
5+
"typeProperties": {
6+
"connectionString": "DefaultEndpointsProtocol=https;AccountName=<account name>;AccountKey=<Account key>"
77
}
8-
}
8+
}
9+
}

‎Samples/ADFWalkthrough/OnPremises/EgressDataToOnPremPipeline.json‎

Lines changed: 39 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,40 @@
11
{
2-
"name": "EgressDataToOnPremPipeline",
3-
"properties":
4-
{
5-
"description" : "Egress data to on-premises SQL Server (v1.0)",
6-
"activities":
7-
[
8-
{
9-
"name": "EgressDataToSqlServer",
10-
"description": "Push Regional Effectiveness Campaign data to On-Premises SQL Server",
11-
"type": "CopyActivity",
12-
"inputs": [ {"name": "MarketingCampaignEffectivenessBlobTable"} ],
13-
"outputs": [ {"name": "MarketingCampaignEffectivenessOnPremSQLTable"} ],
14-
"transformation":
15-
{
16-
"source":
17-
{
18-
"type": "BlobSource"
19-
},
20-
"sink":
21-
{
22-
"type": "SqlSink",
23-
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
24-
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
25-
}
26-
},
27-
"Policy":
28-
{
29-
"concurrency": 1,
30-
"executionPriorityOrder": "NewestFirst",
31-
"style": "StartOfInterval",
32-
"retry": 0,
33-
"timeout": "01:00:00"
34-
}
35-
}
36-
]
37-
}
38-
}
2+
"name": "EgressDataToOnPremPipeline",
3+
"properties": {
4+
"description": "Egress data to on-premises SQL Server (v1.0)",
5+
"activities": [
6+
{
7+
"name": "EgressDataToSqlServer",
8+
"description": "Push Regional Effectiveness Campaign data to On-Premises SQL Server",
9+
"type": "Copy",
10+
"inputs": [
11+
{
12+
"name": "MarketingCampaignEffectivenessBlobTable"
13+
}
14+
],
15+
"outputs": [
16+
{
17+
"name": "MarketingCampaignEffectivenessOnPremSQLTable"
18+
}
19+
],
20+
"typeProperties": {
21+
"source": {
22+
"type": "BlobSource"
23+
},
24+
"sink": {
25+
"type": "SqlSink",
26+
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
27+
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
28+
}
29+
},
30+
"Policy": {
31+
"concurrency": 1,
32+
"executionPriorityOrder": "NewestFirst",
33+
"style": "StartOfInterval",
34+
"retry": 0,
35+
"timeout": "01:00:00"
36+
}
37+
}
38+
]
39+
}
40+
}

‎Samples/ADFWalkthrough/OnPremises/MarketingCampaignEffectivenessOnPremSQLTable.json‎

Lines changed: 57 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,60 @@
11
{
2-
"name": "MarketingCampaignEffectivenessOnPremSQLTable",
3-
"properties":
4-
{
5-
"structure":
6-
[
7-
{ "name": "ProfileID", "type": "String"},
8-
{ "name": "SessionStart", "type": "String"},
9-
{ "name": "Duration", "type": "Int"},
10-
{ "name": "State", "type": "String"},
11-
{ "name": "SrcIPAddress", "type": "String"},
12-
{ "name": "GameType", "type": "String"},
13-
{ "name": "Multiplayer", "type": "String"},
14-
{ "name": "EndRank", "type": "Int"},
15-
{ "name": "WeaponsUsed", "type": "Int"},
16-
{ "name": "UsersInteractedWith", "type": "String"},
17-
{ "name": "Impressions", "type": "String"}
18-
],
19-
"location":
20-
{
21-
"type": "OnPremisesSqlServerTableLocation",
22-
"tableName": "MarketingCampaignEffectiveness",
23-
"linkedServiceName": "OnPremSqlServerLinkedService"
24-
},
25-
"availability":
26-
{
27-
"frequency": "Day",
28-
"interval": 1
29-
}
2+
"name": "MarketingCampaignEffectivenessOnPremSQLTable",
3+
"properties": {
4+
"structure": [
5+
{
6+
"name": "ProfileID",
7+
"type": "String"
8+
},
9+
{
10+
"name": "SessionStart",
11+
"type": "String"
12+
},
13+
{
14+
"name": "Duration",
15+
"type": "Int"
16+
},
17+
{
18+
"name": "State",
19+
"type": "String"
20+
},
21+
{
22+
"name": "SrcIPAddress",
23+
"type": "String"
24+
},
25+
{
26+
"name": "GameType",
27+
"type": "String"
28+
},
29+
{
30+
"name": "Multiplayer",
31+
"type": "String"
32+
},
33+
{
34+
"name": "EndRank",
35+
"type": "Int"
36+
},
37+
{
38+
"name": "WeaponsUsed",
39+
"type": "Int"
40+
},
41+
{
42+
"name": "UsersInteractedWith",
43+
"type": "String"
44+
},
45+
{
46+
"name": "Impressions",
47+
"type": "String"
48+
}
49+
],
50+
"type": "SqlServerTable",
51+
"linkedServiceName": "OnPremSqlServerLinkedService",
52+
"typeProperties": {
53+
"tableName": "MarketingCampaignEffectiveness"
54+
},
55+
"availability": {
56+
"frequency": "Day",
57+
"interval": 1
3058
}
59+
}
3160
}

‎Samples/ADFWalkthrough/OnPremises/OnPremSqlServerLinkedService.json‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
{
22
"Name": "OnPremSqlServerLinkedService",
3-
"Properties":
4-
{
5-
"type": "OnPremisesSqlLinkedService",
6-
"ConnectionString": "Data Source=<servername>;Initial Catalog=MarketingCampaigns;Integrated Security=False;User ID=<username>;Password=<password>;",
7-
"gatewayName": "<gatewayname>"
3+
"properties": {
4+
"type": "OnPremisesSqlServer",
5+
"typeProperties": {
6+
"ConnectionString": "Data Source=<servername>;Initial Catalog=MarketingCampaigns;Integrated Security=False;User ID=<username>;Password=<password>;",
7+
"gatewayName": "<gatewayname>"
8+
}
89
}
9-
}
10+
}

‎Samples/ADFWalkthrough/Pipelines/AnalyzeMarketingCampaignPipeline.json‎

Lines changed: 72 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,74 @@
11
{
2-
"name": "AnalyzeMarketingCampaignPipeline",
3-
"properties":
4-
{
5-
"description" : "(v1.0) Join the Regional Campaign data and with Enriched Gamer Fact Data and push to Azure SQLD Database",
6-
"activities":
7-
[
8-
{
9-
"name": "JoinData",
10-
"description": "Join Regional Campaign data with Enriched Gamer Fact Data",
11-
"type": "HDInsightActivity",
12-
"inputs": [ {"name": "EnrichedGameEventsTable"}, {"name": "RefMarketingCampaignTable"} ],
13-
"outputs": [ {"name": "MarketingCampaignEffectivenessBlobTable"} ],
14-
"linkedServiceName": "HDInsightLinkedService",
15-
"transformation":
16-
{
17-
"type": "Hive",
18-
"extendedProperties":
19-
{
20-
"EventsInput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/logs/enrichedgameevents/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)",
21-
"CampaignInput": "wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/refdata/refmarketingcampaign/",
22-
"CampaignOutput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/marketingcampaigneffectiveness/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)"
23-
},
24-
"scriptpath": "adfwalkthrough\\scripts\\transformdata.hql",
25-
"scriptLinkedService": "StorageLinkedService"
26-
},
27-
"policy":
28-
{
29-
"concurrency": 1,
30-
"executionPriorityOrder": "NewestFirst",
31-
"retry": 1,
32-
"timeout": "01:00:00"
33-
}
34-
},
35-
{
36-
"name": "EgressDataAzure",
37-
"description": "Push Regional Effectiveness Campaign data to Sql Azure",
38-
"type": "CopyActivity",
39-
"inputs": [ {"name": "MarketingCampaignEffectivenessBlobTable"} ],
40-
"outputs": [ {"name": "MarketingCampaignEffectivenessSQLTable"} ],
41-
"transformation":
42-
{
43-
"source":
44-
{
45-
"type": "BlobSource"
46-
},
47-
"sink":
48-
{
49-
"type": "SqlSink",
50-
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
51-
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
52-
}
53-
},
54-
"Policy":
55-
{
56-
"concurrency": 1,
57-
"executionPriorityOrder": "NewestFirst",
58-
"style": "StartOfInterval",
59-
"retry": 0,
60-
"timeout": "01:00:00"
61-
}
62-
}
63-
]
2+
"name": "AnalyzeMarketingCampaignPipeline",
3+
"properties": {
4+
"description": "(v1.0) Join the Regional Campaign data and with Enriched Gamer Fact Data and push to Azure SQLD Database",
5+
"activities": [
6+
{
7+
"name": "JoinData",
8+
"description": "Join Regional Campaign data with Enriched Gamer Fact Data",
9+
"inputs": [
10+
{
11+
"name": "EnrichedGameEventsTable"
12+
},
13+
{
14+
"name": "RefMarketingCampaignTable"
15+
}
16+
],
17+
"outputs": [
18+
{
19+
"name": "MarketingCampaignEffectivenessBlobTable"
20+
}
21+
],
22+
"linkedServiceName": "HDInsightLinkedService",
23+
"type": "HDInsightHive",
24+
"typeProperties": {
25+
"scriptpath": "adfwalkthrough\\scripts\\transformdata.hql",
26+
"scriptLinkedService": "StorageLinkedService",
27+
"defines": {
28+
"EventsInput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/logs/enrichedgameevents/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)",
29+
"CampaignInput": "wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/refdata/refmarketingcampaign/",
30+
"CampaignOutput": "$$Text.Format('wasb://adfwalkthrough@<storageaccountname>.blob.core.windows.net/marketingcampaigneffectiveness/yearno={0:yyyy}/monthno={0:%M}/dayno={0:%d}/', SliceStart)"
31+
}
32+
},
33+
"policy": {
34+
"concurrency": 1,
35+
"executionPriorityOrder": "NewestFirst",
36+
"retry": 1,
37+
"timeout": "01:00:00"
38+
}
39+
},
40+
{
41+
"name": "EgressDataAzure",
42+
"description": "Push Regional Effectiveness Campaign data to Sql Azure",
43+
"type": "Copy",
44+
"inputs": [
45+
{
46+
"name": "MarketingCampaignEffectivenessBlobTable"
47+
}
48+
],
49+
"outputs": [
50+
{
51+
"name": "MarketingCampaignEffectivenessSQLTable"
52+
}
53+
],
54+
"typeProperties": {
55+
"source": {
56+
"type": "BlobSource"
57+
},
58+
"sink": {
59+
"type": "SqlSink",
60+
"SqlWriterTableType": "MarketingCampaignEffectivenessType",
61+
"SqlWriterStoredProcedureName": "spEgressOverwriteMarketingCampaignEffectiveness"
62+
}
63+
},
64+
"Policy": {
65+
"concurrency": 1,
66+
"executionPriorityOrder": "NewestFirst",
67+
"style": "StartOfInterval",
68+
"retry": 0,
69+
"timeout": "01:00:00"
70+
}
6471
}
65-
}
72+
]
73+
}
74+
}

0 commit comments

Comments
 (0)