Skip to content

Commit 3fbdc14

Browse files
committed
Merge remote-tracking branch 'refs/remotes/Microsoft/master'
2 parents 12de455 + 804a4d8 commit 3fbdc14

File tree

5 files changed

+104
-60
lines changed

5 files changed

+104
-60
lines changed

samples/databases/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,8 @@ The new sample database for SQL Server 2016 and Azure SQL Database. It illustrat
99
__[contoso-data-warehouse](contoso-data-warehouse/)__
1010

1111
Sample data warehouse that illustrates loading data into Azure SQL Data Warehouse.
12+
13+
__[adventure-works-lt](adventure-works-lt/)__
14+
15+
AdventureWorksLT sample database.
16+
366 KB
Binary file not shown.

samples/manage/azure-automation-automated-export/AutoExport.ps1

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,16 @@ Add-Type -TypeDefinition @"
1818
$databaseServerPairs =
1919
@([pscustomobject]@{serverName="SAMPLESERVER1";databaseName="SAMPLEDATABASE1"},
2020
[pscustomobject]@{serverName="SAMPLESERVER1";databaseName="SAMPLEDATABASE2"},
21-
[pscustomobject]@{serverName="SAMPLESERVER2";databaseName="SAMPLEDATABASE3"});
21+
[pscustomobject]@{serverName="SAMPLESERVER2";databaseName="SAMPLEDATABASE3"}
22+
);
2223

23-
$serverCred = Get-AutomationPSCredential -Name 'NAMEOFSERVERCREDENTIAL1';
24+
# The Credentials for the database servers
25+
$serverCred1 = Get-AutomationPSCredential -Name 'NAMEOFSERVERCREDENTIAL1';
2426
$serverCred2 = Get-AutomationPSCredential -Name 'NAMEOFSERVERCREDENTIAL2';
25-
$serverCredentialsDictionary = @{'SAMPLESERVER1'=$serverCred;'SAMPLESERVER2'=$serverCred2}
27+
$serverCredentialsDictionary = @{
28+
'SAMPLESERVER1'=$serverCred1;
29+
'SAMPLESERVER2'=$serverCred2;
30+
}
2631

2732
# The number of databases you want to have running at the same time.
2833
$batchingLimit = 10;
@@ -31,11 +36,12 @@ $retryLimit = 5;
3136
# The number of minutes you want to wait for an operation to finish before you fail.
3237
$waitInMinutes = 30;
3338

39+
# Connection Asset Name for Authenticating (Keep as AzureClassicRunAsConnection if you created the default RunAs accounts)
40+
$connectionAssetName = "AzureClassicRunAsConnection";
41+
3442
$storageKeyVariableName = "STORAGEKEYVARIABLENAME";
3543
$storageAccountName = "STORAGEACCOUNTNAME";
36-
$automationCertificateName = "CERTIFICATENAME";
37-
$subId = "00000000-0000-0000-0000-000000000000";
38-
$subName = "SUBSCRIPTIONNAME";
44+
$storageContainerName = "STORAGECONTAINERNAME";
3945

4046
function LogMessage($message)
4147
{
@@ -133,21 +139,19 @@ function CheckCopy($dbObj)
133139
# This function starts the export. If there is an error, we set the state to ToDrop. Otherwise, we set the state to Exporting.
134140
function StartExport($dbObj)
135141
{
136-
# Setup the server connection that the storage account is on.
137-
$serverManageUrl = "https://autoexportserver.database.windows.net";
138142
# Get the current time to use as a unique identifier for the blob name.
139143
$currentTime = Get-Date -format "_yyyy-MM-dd_HH:mm.ss";
140144
$blobName = $dbObj.DatabaseName + "_ExportBlob" + $currentTime;
141145
# Use the stored credential to create a server credential to use to login to the server.
142-
$servercredential = $global:serverCredentialsDictionary[$dbObj.ServerName];
146+
$serverCredential = $global:serverCredentialsDictionary[$dbObj.ServerName];
143147
# Set up a SQL connection context to use when exporting.
144-
$ctx = New-AzureSqlDatabaseServerContext -ServerName $dbObj.ServerName -Credential $servercredential;
148+
$ctx = New-AzureSqlDatabaseServerContext -ServerName $dbObj.ServerName -Credential $serverCredential;
145149
# Get the storage key to setup the storage context.
146150
$storageKey = Get-AutomationVariable -Name $global:storageKeyVariableName;
147151
# Get the storage context.
148152
$stgctx = New-AzureStorageContext -StorageAccountName $global:storageAccountName -StorageAccountKey $storageKey;
149153
# Start the export. If there is an error, stop the export and set the state to ToDrop.
150-
$dbObj.Export = Start-AzureSqlDatabaseExport -SqlConnectionContext $ctx -StorageContext $stgctx -StorageContainerName autoexportcontainer -DatabaseName $dbObj.DatabaseCopyName -BlobName $blobName;
154+
$dbObj.Export = Start-AzureSqlDatabaseExport -SqlConnectionContext $ctx -StorageContext $stgctx -StorageContainerName $global:storageContainerName -DatabaseName $dbObj.DatabaseCopyName -BlobName $blobName;
151155
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
152156
if (-not $? -and $global:retryLimit -ile $dbObj.RetryCount)
153157
{
@@ -166,7 +170,7 @@ function StartExport($dbObj)
166170
}
167171
# Set the state to Exporting.
168172
$dbObj.DatabaseState = ([DatabaseState]::Exporting);
169-
LogMessage ("Exporting " + $dbObj.DatabaseCopyName);
173+
LogMessage ("Exporting " + $dbObj.DatabaseCopyName + " with RequestID: " + $dbObj.Export.RequestGuid);
170174
$dbObj.OperationStartTime = Get-Date;
171175
}
172176

@@ -187,7 +191,7 @@ function CheckExport($dbObj)
187191
{
188192
# If the status is "Failed" and we have more retries left, try to export the database copy again.
189193
LogMessage ("The last export failed on database " + $dbObj.DatabaseName + ", going back to ToExport state to try again");
190-
LogMessage $check
194+
LogMessage $check.ErrorMessage
191195
$dbObj.DatabaseState = ([DatabaseState]::ToExport);
192196
$dbObj.RetryCount++;
193197
return;
@@ -224,7 +228,7 @@ function ExportProcess
224228
$dbsToCopy = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::ToCopy);
225229
for($i = 0; $i -lt $dbsToCopy.Count; $i++)
226230
{
227-
LogMessage $dbsToCopy[$i];
231+
LogMessage "Database Name: $($dbsToCopy[$i].DatabaseName) State: $($dbsToCopy[$i].DatabaseState) Retry Count: $($dbsToCopy[$i].RetryCount)";
228232
StartCopy($dbsToCopy[$i]);
229233
}
230234

@@ -239,7 +243,7 @@ function ExportProcess
239243
$dbsToExport = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::ToExport);
240244
for($i = 0; $i -lt $dbsToExport.Count; $i++)
241245
{
242-
LogMessage $dbsToExport[$i];
246+
LogMessage "Database Name: $($dbsToExport[$i].DatabaseName) State: $($dbsToExport[$i].DatabaseState) Retry Count: $($dbsToExport[$i].RetryCount)";
243247
StartExport($dbsToExport[$i]);
244248
}
245249

@@ -254,7 +258,7 @@ function ExportProcess
254258
$dbsToDrop = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::ToDrop);
255259
for($i = 0; $i -lt $dbsToDrop.Count; $i++)
256260
{
257-
LogMessage $dbsToDrop[$i];
261+
LogMessage "Database Name: $($dbsToDrop[$i].DatabaseName) State: $($dbsToDrop[$i].DatabaseState) Retry Count: $($dbsToDrop[$i].RetryCount)";
258262
StartDrop($dbsToDrop[$i]);
259263
}
260264

@@ -266,11 +270,25 @@ function ExportProcess
266270
}
267271
}
268272

269-
# Get the certificate to authenticate the subscription
270-
$cert = Get-AutomationCertificate -Name $global:automationCertificateName;
271-
# Set the subscription to use
272-
Set-AzureSubscription -SubscriptionName $global:subName -Certificate $cert -SubscriptionId $global:subID;
273-
Select-AzureSubscription -Current $global:subName;
273+
# Authenticate to Azure with certificate
274+
Write-Verbose "Get connection asset: $connectionAssetName" -Verbose;
275+
$automationConnection = Get-AutomationConnection -Name $connectionAssetName;
276+
if ($automationConnection -eq $null)
277+
{
278+
throw "Could not retrieve connection asset: $connectionAssetName. Assure that this asset exists in the Automation account.";
279+
}
280+
281+
$certificateAssetName = $automationConnection.CertificateAssetName;
282+
Write-Verbose "Getting the certificate: $certificateAssetName" -Verbose;
283+
$automationCertificate = Get-AutomationCertificate -Name $certificateAssetName;
284+
if ($automationCertificate -eq $null)
285+
{
286+
throw "Could not retrieve certificate asset: $certificateAssetName. Assure that this asset exists in the Automation account.";
287+
}
288+
289+
Write-Verbose "Authenticating to Azure with certificate." -Verbose;
290+
Set-AzureSubscription -SubscriptionName $automationConnection.SubscriptionName -SubscriptionId $automationConnection.SubscriptionID -Certificate $automationCertificate;
291+
Select-AzureSubscription -SubscriptionId $automationConnection.SubscriptionID;
274292

275293
$currentIndex = 0;
276294
for($currentRun = 0; $currentRun -lt ([math]::Ceiling($databaseServerPairs.Length/$batchingLimit)); $currentRun++)

samples/manage/azure-automation-automated-export/AutoExportBlobRetention.ps1

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,21 @@
1-
# The storage key for the storage account you are using.
2-
$storageKey = Get-AutomationVariable -Name "STORAGEKEYVARIABLENAME";
3-
# The name of the storage container you are using.
4-
$storageContainer = "STORAGECONTAINERNAME";
5-
# Set up the storage context for the storage account.
6-
$context = New-AzureStorageContext -StorageAccountName "STORAGEACCOUNTNAME" -StorageAccountKey $storageKey
7-
# Get all of the blobs in the storage account.
8-
$blobs = Get-AzureStorageBlob -Container $storageContainer -Context $context
1+
#Azure Automation String Variable name for your Storage Account Key
2+
$storageKeyVariableName = "STORAGEKEYVARIABLENAME";
3+
#Name of your Storage Account
4+
$storageAccountName = "STORAGEACCOUNTNAME";
5+
#Name of your Storage Container
6+
$storageContainerName = "STORAGECONTAINERNAME";
97
# Set the number of days that you want the blob to be stored for.
108
$retentionInDays = 30
119

10+
11+
# Get the storage key
12+
$storageKey = Get-AutomationVariable -Name $storageKeyVariableName;
13+
# Set up the storage context for the storage account.
14+
$context = New-AzureStorageContext -StorageAccountName $storageAccountName -StorageAccountKey $storageKey
15+
# Get all of the blobs in the storage account.
16+
$blobs = Get-AzureStorageBlob -Container $storageContainerName -Context $context
17+
18+
1219
foreach($blob in $blobs)
1320
{
1421
# Get the current time to compare to the time that the blob was created.

samples/manage/azure-automation-automated-export/README.md

Lines changed: 45 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,23 @@ author: trgrie-msft
88

99
Provides the scripts and lists the steps to set up automatically exporting your databases to Azure Storage with Azure Automation.
1010

11-
## Azure Automation Set Up
12-
13-
1. Create and uploade the certificates that you will use to authenticate your connection to azure.
14-
- Run powershell as admin.
15-
- Run the New-SelfSignedCertificate command: New-SelfSignedCertificate -CertStoreLocation cert:\localmachine\my -DnsName <certificateName>
16-
- Create a corresponding pfx certificate by taking the thumbprint of the newly created certificate and running these commands:
17-
- $CertPassword = ConvertTo-SecureString -String <YourPassword> -Force -AsPlainText
18-
- Export-PfxCertificate -Cert cert:\localmachine\my\<thumbprint> -FilePath <PathAndFileName>.pfx -Password $CertPassword
19-
- Upload the .cer file to your subscription [here][https://manage.windowsazure.com/]
20-
- Upload the .pfx file to the certificates under Assets in the automation account that you want to use on Azure. You will use the password you gave in the previous step to authenticate it.
21-
2. Create new a new credentials asset to authenticate your server with.
22-
- Under assets, click on Credentials, and then click on Add a credential.
23-
- Name the credential and give the username and password that you will be logging into the server with.
24-
3. Create a new variable asset to pass the storage key of the Azure storage account you will be using.
25-
- Under assets, click on variables and then Add a variable.
26-
- Give the value of the storage key and you can make it encrypted so that only Azure Automation can read the variable and it won't show the key in plaintext if someone looks at the variable.
27-
4. Set Up Log Analytics (OMS) and Alerts
28-
- If you don't have Log Analytics set up on your Azure account, follow [these][https://azure.microsoft.com/en-us/documentation/articles/automation-manage-send-joblogs-log-analytics/] instructions for setting it up.
11+
## Prerequisite Set Up
12+
13+
1. Create and set up your Azure Automation Account
14+
- Create an Azure Automation Account by [following the instructions here](https://docs.microsoft.com/en-us/azure/automation/automation-sec-configure-azure-runas-account).
15+
2. Add Azure Automation Credential assets for your SQL Azure servers
16+
- Create your Automation Credential for each of your SQL Azure servers you intend to export by [following the instructions here](https://docs.microsoft.com/en-us/azure/automation/automation-credentials#creating-a-new-credential-asset).
17+
3. Create the Azure Storage Account to hold your bacpac files
18+
- Create the Storage Account by [following the instructions here](https://docs.microsoft.com/en-us/azure/storage/storage-create-storage-account#create-a-storage-account).
19+
- Create the Blob Storage Container
20+
- Go to your Storage Account
21+
- Click the Blobs tile
22+
- Click the Add Container button
23+
- Name the container, keep the access type as Private, and click the Create button
24+
- Copy your Storage Account access keys by [following the instructions here](https://docs.microsoft.com/en-us/azure/storage/storage-create-storage-account#view-and-copy-storage-access-keys).
25+
- Create an Azure Automation string Variable asset for your Storage Account access key by [following the instructions here](https://docs.microsoft.com/en-us/azure/automation/automation-variables#creating-an-automation-variable).
26+
4. Set Up Log Analytics (OMS) and Alerts (optional for alerting)
27+
- If you don't have Log Analytics set up on your Azure account, [follow these](https://docs.microsoft.com/en-us/azure/automation/automation-manage-send-joblogs-log-analytics) instructions for setting it up.
2928
5. Set Up Log Analytics Alerts
3029
- To send yourself an email if an error occurs or one of the jobs fails, you need to set up alerts.
3130
- Select your log analytics account that you want to use in the azure portal and click on the OMS Portal box under Management.
@@ -36,18 +35,33 @@ Provides the scripts and lists the steps to set up automatically exporting your
3635

3736
## Script Set Up
3837

38+
Save the AutoExport.ps1 and AutoExportBlobRetention.ps1 files locally to make these edits
39+
3940
1. In the AutoExport.ps1 script, here are the values that need to be modified:
40-
- $databaseServerPairs: This is where you put in the names of the databases you want to export along with the name of the server they are on.
41-
- $serverCredentialsDictionary: If you are backing up from multiple servers, you can setup all of the credentials here and look them up by the server’s name later.
42-
- $batchingLimit: This tells the script how many databases can be worked on at the same time (basically, the maximum number of database copies that there will be at once).
43-
- $retryLimit: This tells the script how many times it can retry an operation.
44-
- $waitTimeInMinutes: This tells the script how long it can wait for an operation to complete before it fails.
45-
- $storageKeyVariableName: This is the AutomationAccount you created the StorageKey variable under (probably the same one you are running the RunBook under) and -Name is the name of the variable.
46-
- $storageAccountName: This is the name of the storage account you are exporting to.
47-
- $automationCertificateName for Get-AutomationCertificate: This is the name of the certificate you setup to authenticate with Azure.
48-
- $subId: The ID of the subscription you are using. This will be used to tell Azure Automation which subscription to use.
49-
- $subName: The name of the subscription you are using. This will be used to tell Azure Automation which subscription to use.
41+
- **$databaseServerPairs:** This is where you put in the names of the databases you want to export along with the name of the server they are on.
42+
Add them in the format: `[pscustomobject]@{serverName="SAMPLESERVER1";databaseName="SAMPLEDATABASE1"}` make sure to comma separate the items
43+
- **$serverCredentialsDictionary:** If you are backing up from multiple servers, you can setup all of the credentials here and look them up by the server’s name later.
44+
Add a $serverCred variable in the format `$serverCred1 = Get-AutomationPSCredential -Name 'NAMEOFSERVERCREDENTIAL1';` for each Azure Automation Credential you created. Increment the variable name (eg. $serverCred2 $serverCred3) for each one.
45+
Add the $serverCreds to the dictionary in the format `'SAMPLESERVERNAME1'=$serverCred1;`
46+
- **$batchingLimit:** This tells the script how many databases can be worked on at the same time (basically, the maximum number of database copies that there will be at once).
47+
- **$retryLimit:** This tells the script how many times it can retry an operation.
48+
- **$waitTimeInMinutes:** This tells the script how long it can wait for an operation to complete before it fails.
49+
- **$storageKeyVariableName:** This is the Azure Automation string Variable name you created to store your Storage Key.
50+
- **$storageAccountName:** This is the name of the storage account you are exporting to.
51+
- **$connectionAssetName:** Connection Asset Name for Authenticating (Keep as AzureClassicRunAsConnection if you created the default RunAs accounts)
5052
2. In AutoExportBlobRetention, here are the values that need to be modified:
51-
- -Name for Get-AzureAutomationVariable: This is the AutomationAccount you created the StorageKey variable under (probably the same one you are running the RunBook under) and -Name is the name of the variable.
52-
- $storageContainer: This is the name of the storage container where you will be monitoring the exported blobs.
53-
- $retentionInDays: This is how many days you want to keep the exported blobs stored for before deleting.
53+
- **$storageKeyVariableName:** This is the Azure Automation string Variable name you created to store your Storage Key.
54+
- **$storageAccountName:** This is the name of your Storage Account you exported your bacpacs to.
55+
- **$storageContainerName:** This is the name of the storage container where you will be monitoring the exported blobs.
56+
- **retentionInDays:** This is how many days you want to keep the exported blobs stored for before deleting.
57+
58+
## Adding the Script to Azure Automation
59+
60+
1. Import the scripts as Azure Automation Runbooks
61+
- Create runbooks from the scripts you editted above by [following the instructions here](https://docs.microsoft.com/en-us/azure/automation/automation-creating-importing-runbook#to-import-a-runbook-from-a-file-with-the-azure-portal) for both scripts.
62+
- [Make sure to publish the runbook.](https://docs.microsoft.com/en-us/azure/automation/automation-creating-importing-runbook#to-publish-a-runbook-using-the-azure-portal)
63+
2. Add a schedule for your Automated Export runbook
64+
- Create a recurring schedule by [following the instructions here](https://docs.microsoft.com/en-us/azure/automation/automation-schedules#to-create-a-new-schedule-in-the-azure-portal).
65+
- Link the schedule(s) you created to the runbooks by [following the instructions here](https://docs.microsoft.com/en-us/azure/automation/automation-schedules#to-link-a-schedule-to-a-runbook-with-the-azure-portal).
66+
67+
You should now be all set up for Automated Exports into blob storage of your selected SQL Azure databases.

0 commit comments

Comments
 (0)