diff --git a/.gitignore b/.gitignore index 9fc58ca..61fdd38 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,252 @@ -################################################################################ -# This .gitignore file was automatically created by Microsoft(R) Visual Studio. -################################################################################ - -/src/XEL2OMS/.vs/XEL2OMS/v14 -/src/XEL2OMS/bin/Debug -/src/XEL2OMS/obj/Debug -/src/XEL2OMS/packages -/src/XEL2OMS/node_modules -/src/XEL2OMS/.vs/config -/src/XEL2OMS/bin -/src/XEL2OMS/obj -/src/XEL2OMS/XEL2OMS.csproj.user +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..42c7ae5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,76 @@ +# Contributing to Azure-SQL-DB-auditing-OMS-integration + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + + - [Code of Conduct](#coc) + - [Issues and Bugs](#issue) + - [Feature Requests](#feature) + - [Submission Guidelines](#submit) + +## Code of Conduct +Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +## Found an Issue? +If you find a bug in the source code or a mistake in the documentation, you can help us by +[submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can +[submit a Pull Request](#submit-pr) with a fix. + +## Want a Feature? +You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub +Repository. If you would like to *implement* a new feature, please submit an issue with +a proposal for your work first, to be sure that we can use it. + +* **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). + +## Submission Guidelines + +### Submitting an Issue +Before you submit an issue, search the archive, maybe your question was already answered. + +If your issue appears to be a bug, and hasn't been reported, open a new issue. +Help us to maximize the effort we can spend fixing issues and adding new +features, by not reporting duplicate issues. Providing the following information will increase the +chances of your issue being dealt with quickly: + +* **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps +* **Version** - what version is affected (e.g. 0.1.2) +* **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you +* **Browsers and Operating System** - is this a problem with all browsers? +* **Reproduce the Error** - provide a live example or a unambiguous set of steps +* **Related Issues** - has a similar issue been reported before? +* **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be + causing the problem (line of code or commit) + +You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/Microsoft/Azure-SQL-DB-auditing-OMS-integration/issues/new. + +### Submitting a Pull Request (PR) +Before you submit your Pull Request (PR) consider the following guidelines: + +* Search the repository (https://github.com/Microsoft/Azure-SQL-DB-auditing-OMS-integration/pulls) for an open or closed PR + that relates to your submission. You don't want to duplicate effort. + +* Make your changes in a new git fork: + +* Commit your changes using a descriptive commit message +* Push your fork to GitHub: +* In GitHub, create a pull request +* If we suggest changes then: + * Make the required updates. + * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): + + ```shell + git rebase master -i + git push -f + ``` + +That is it! Thank you for your contribution! \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d1ca00f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE \ No newline at end of file diff --git a/README.md b/README.md index 4a8dc42..5fa185c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#Azure SQL DB Auditing log integration into OMS +# Azure SQL DB Auditing log integration into OMS This is a **sync application** that runs in Azure and utilizes OMS public APIs to push SQL audit logs into OMS. @@ -13,7 +13,7 @@ It allows using OMS Log Analytics to explore and analyze your database activity, 2. Azure Subscription with resource creation permissions 3. OMS workspace with Administrator or Contributor permissions -#Estimated Cost of Deployed Resources +# Estimated Cost of Deployed Resources | Resource | Cost/Month | Cost/Hr | @@ -21,10 +21,10 @@ It allows using OMS Log Analytics to explore and analyze your database activity, | [B1 App Service Plan](https://azure.microsoft.com/en-us/pricing/details/app-service/) | $55.80 | $0.075 | | [Storage Plan](https://azure.microsoft.com/en-us/pricing/details/storage/) | ~$0 | $0.0036 / transaction | -#Setup Guide +# Setup Guide -###Retrieve SQL DB Auditing - Storage Connection String +### Retrieve SQL DB Auditing - Storage Connection String 1. Launch the [Azure Portal](https://portal.azure.com) at https://portal.azure.com. @@ -33,7 +33,8 @@ It allows using OMS Log Analytics to explore and analyze your database activity, ![Navigation Pane][1]
-###Retrieve OMS Workspace ID and Access key + +### Retrieve OMS Workspace ID and Access key 1. Launch the [Microsoft Operations Management Suite (OMS)](https://mms.microsoft.com) at https://mms.microsoft.com. @@ -48,13 +49,14 @@ It allows using OMS Log Analytics to explore and analyze your database activity, ![Navigation Pane][3]
-###Deploy sync application to Azure + +### Deploy sync application to Azure 1. Click on the **Deploy to Azure** button below to initiate deployment process. > During deployment, use the **Storage Connection String**, **Workspace ID**, and **Primary Key** that you saved in the previous steps. - + [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FMicrosoft%2FAzure-SQL-DB-auditing-OMS-integration%2Fmaster%2Fazuredeploy.json) 2. When deployment is completed, you can close the web application browser window. @@ -64,7 +66,8 @@ It allows using OMS Log Analytics to explore and analyze your database activity, > Once the sync application is deployed in Azure, it can take up to 5-10 minutes for initial data to start appearing in your OMS workspace.
-###Import Azure SQL DB audit sync dashboard to OMS + +### Import Azure SQL DB audit sync dashboard to OMS 1. Download [SQLDatabaseAudit.omsview][101] to your PC. @@ -86,6 +89,7 @@ It allows using OMS Log Analytics to explore and analyze your database activity,
+ # Troubleshooting > Once the sync application is deployed in Azure, it can take up to 5-10 minutes for initial data to start appearing in your OMS workspace. @@ -114,6 +118,4 @@ If you've completed the setup process but don't see audit data in your OMS works [9]: ./media/9_webjobs_logs.png [10]: ./media/10_webjobs_logs_2.png -[101]: https://github.com/Microsoft/Azure-SQL-DB-auditing-OMS-integration/blob/master/SQLDatabaseAudit.omsview - - +[101]: https://github.com/Microsoft/Azure-SQL-DB-auditing-OMS-integration/blob/master/SQLDatabaseAudit.omsview \ No newline at end of file diff --git a/azuredeploy.json b/azuredeploy.json index 45b7684..99ad866 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -1,91 +1,61 @@ { - "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { "siteName": { - "type": "string" - }, - - "siteLocation": { - - "type": "string" - - }, - + "siteLocation": { + "type": "string" + }, "SQL Auditing - Storage Connection String": { - "type": "string", - "metadata": { - "description": "Storage connection string provides access to the Azure storage account that contains SQL audit logs in blob storage." - } - }, - + "OmsEndpointAddress": { + "type": "string", + "defaultValue": "ods.opinsights.azure.com", + "allowedValues": [ + "ods.opinsights.azure.com", + "ods.opinsights.azure.us" + ], + "metadata": { + "description": "OMS endpoint where the log should be shipped." + } + }, "OMS Workspace Id": { - "type": "string", - "metadata": { - "description": "OMS Workspace ID indicates the Microsoft Operations Management Suite account that provides log analytics services." - } - }, - "OMS Workspace Key": { - "type": "string", - "metadata": { - "description": "The OMS Access Key provides access to the Microsoft Operations Management Suite account that provides log analytics services." - } - }, "repoUrl": { - - "type": "string" - + "type": "string", + "defaultValue": "https://github.com/Microsoft/Azure-SQL-DB-auditing-OMS-integration" }, - "branch": { - - "type": "string" - + "type": "string", + "defaultValue": "master" } }, - "variables": { - "hostingPlanName": "SQLAuditLogsToOMSPlan", - "jobCollectionName": "SQLAuditLogsToOMSJob", - "ContainerName": "sqldbauditlogs" - - }, - "resources": [ - { - "apiVersion": "2015-08-01", - "name": "[variables('hostingPlanName')]", - "type": "Microsoft.Web/serverfarms", - "location": "[parameters('siteLocation')]", "sku": { "name": "F1", @@ -94,205 +64,107 @@ "family": "F", "capacity": 0 }, - "properties": { - "name": "[variables('hostingPlanName')]", - "numberOfWorkers": 0 - } - }, - { - "apiVersion": "2015-08-01", - "name": "[parameters('siteName')]", - "type": "Microsoft.Web/sites", - "location": "[parameters('siteLocation')]", - "dependsOn": [ - "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]" - ], - "properties": { - "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]" - }, - "resources": [ - { - "apiVersion": "2015-08-01", - "name": "web", - "type": "sourcecontrols", - "dependsOn": [ - "[resourceId('Microsoft.Web/Sites', parameters('siteName'))]" - ], - "properties": { - "RepoUrl": "[parameters('repoUrl')]", - "branch": "[parameters('branch')]", - "IsManualIntegration": true - } - }, - { - "apiVersion": "2015-08-01", - "name": "appsettings", - "type": "config", - "dependsOn": [ - "[resourceId('Microsoft.Web/Sites', parameters('siteName'))]", "[resourceId('Microsoft.Web/sites/sourcecontrols', parameters('siteName'), 'web')]" - ], - "properties": { - "ConnectionString": "[parameters('SQL Auditing - Storage Connection String')]", - "ContainerName": "[variables('ContainerName')]", - + "OmsEndpointAddress": "[parameters('OmsEndpointAddress')]", "omsWorkspaceId": "[parameters('OMS Workspace Id')]", - "omsWorkspaceKey": "[parameters('OMS Workspace Key')]" - } - } - ] - }, - { - "apiVersion": "2016-03-01", - "name": "[variables('jobCollectionName')]", - "type": "Microsoft.Scheduler/jobCollections", - "dependsOn": [ - "[resourceId('Microsoft.Web/Sites', parameters('siteName'))]", "[resourceId('Microsoft.Web/sites/sourcecontrols', parameters('siteName'), 'web')]" - ], - "location": "[parameters('siteLocation')]", - "properties": { - "sku": { - "name": "standard" - }, - "quota": { - "maxJobCount": "10", - "maxRecurrence": { - "Frequency": "minute", - "interval": "1" - } - } - }, - "resources": [ - { - "apiVersion": "2016-03-01", - "name": "SQLAuditLogsToOMSJob", - "type": "jobs", - "dependsOn": [ - "[resourceId('Microsoft.Web/sites/config', parameters('siteName'), 'appsettings')]", "[resourceId('Microsoft.Web/sites/sourcecontrols', parameters('siteName'), 'web')]", - "[resourceId('Microsoft.Scheduler/jobCollections', variables('jobCollectionName'))]" - ], "location": "[parameters('siteLocation')]", - "properties": { - "action": { - "request": { - "uri": "[concat(list(resourceId('Microsoft.Web/sites/config', parameters('siteName'), 'publishingcredentials'), '2014-06-01').properties.scmUri, '/api/triggeredjobs/SQLAuditLogsToOMSJob/run')]", - "method": "POST" - }, - "type": "http", - "retryPolicy": { - "retryType": "Fixed", - "retryInterval": "PT1M", - "retryCount": 2 - } - }, - "state": "enabled", - "recurrence": { - "interval": "15", - "Frequency": "Minute" - } - } - } - ] - } - ] - -} +} \ No newline at end of file diff --git a/src/XEL2OMS/App.config b/src/XEL2OMS/App.config index 3fc55d0..b6a7e54 100644 --- a/src/XEL2OMS/App.config +++ b/src/XEL2OMS/App.config @@ -1,13 +1,13 @@ - + + - - + @@ -23,6 +23,34 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/XEL2OMS/HttpException.cs b/src/XEL2OMS/HttpException.cs index a067d3e..4b5ab3c 100644 --- a/src/XEL2OMS/HttpException.cs +++ b/src/XEL2OMS/HttpException.cs @@ -1,22 +1,43 @@ -using System; -using System.Runtime.Serialization; +// ----------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +// ----------------------------------------------------------------------- namespace XEL2OMS { + using System; + using System.Runtime.Serialization; + + /// + /// The exception that is thrown when a HTTP error occurs. + /// [Serializable] internal class HttpException : Exception { private int statusCode; private string v; + /// + /// Initializes a new instance of the class. + /// public HttpException() { } + /// + /// Initializes a new instance of the class. + /// + /// The message that describes the error. public HttpException(string message) : base(message) { } + /// + /// Initializes a new instance of the class. + /// + /// The error message that explains the reason for the exception. + /// The exception that is the cause of the current exception, or a null reference (Nothing in Visual Basic) if no inner exception is specified. public HttpException(string message, Exception innerException) : base(message, innerException) { } @@ -27,6 +48,11 @@ public HttpException(int statusCode, string v) this.v = v; } + /// + /// Initializes a new instance of the class. + /// + /// The that holds the serialized object data about the exception being thrown. + /// The that contains contextual information about the source or destination. protected HttpException(SerializationInfo info, StreamingContext context) : base(info, context) { } diff --git a/src/XEL2OMS/OmsIngestionApi.cs b/src/XEL2OMS/OmsIngestion.cs similarity index 71% rename from src/XEL2OMS/OmsIngestionApi.cs rename to src/XEL2OMS/OmsIngestion.cs index faa6487..720d4a9 100644 --- a/src/XEL2OMS/OmsIngestionApi.cs +++ b/src/XEL2OMS/OmsIngestion.cs @@ -1,23 +1,28 @@ -using Microsoft.Practices.EnterpriseLibrary.TransientFaultHandling; -using System; -using System.Diagnostics; -using System.IO; -using System.Net; -using System.Text; -using System.Threading.Tasks; -using System.Web; -using System.Globalization; +// ----------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +// ----------------------------------------------------------------------- namespace XEL2OMS { - public class OMSIngestionApi + using System; + using System.Configuration; + using System.Diagnostics; + using System.IO; + using System.Net; + using System.Text; + using System.Threading.Tasks; + using Microsoft.Practices.EnterpriseLibrary.TransientFaultHandling; + + public class OmsIngestion { private readonly string m_CustomerId; private readonly string m_SharedKey; private readonly TraceSource m_Tracer; private readonly RetryPolicy m_Retry; - public OMSIngestionApi(TraceSource tracer, string customerId, string sharedKey) + public OmsIngestion(TraceSource tracer, string customerId, string sharedKey) { // Check the shared key is of a valid format Convert.FromBase64String(sharedKey); @@ -28,24 +33,8 @@ public OMSIngestionApi(TraceSource tracer, string customerId, string sharedKey) m_Retry = RetryPolicy.DefaultFixed; } - private string GetOMSApiSignature(string date, int contentLength) - { - var xHeaders = string.Format("x-ms-date:{0}", date); - var stringToHash = string.Format("POST\n{0}\napplication/json\n{1}\n/api/logs", contentLength, xHeaders); - - var bytesToHash = Encoding.UTF8.GetBytes(stringToHash); - var keyBytes = Convert.FromBase64String(m_SharedKey); - - using (var sha256 = new System.Security.Cryptography.HMACSHA256(keyBytes)) - { - var calculatedHash = sha256.ComputeHash(bytesToHash); - var encodedHash = Convert.ToBase64String(calculatedHash); - var authorization = string.Format("SharedKey {0}:{1}", m_CustomerId, encodedHash); - return authorization; - } - } - public async Task SendOMSApiIngestionFile(string requestBody) + public async Task SendAsync(string requestBody) { var method = "POST"; var contentType = "application/json"; @@ -54,12 +43,12 @@ public async Task SendOMSApiIngestionFile(string requestBody) Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); - string address = string.Format("https://{0}.ods.opinsights.azure.com/api/logs?api-version=2016-04-01", m_CustomerId); + string address = $"https://{m_CustomerId}.{ConfigurationManager.AppSettings["OmsEndpointAddress"]}/api/logs?api-version=2016-04-01"; Uri uriAddress = new Uri(address); byte[] payload = Encoding.UTF8.GetBytes(requestBody); - var signature = GetOMSApiSignature(date, payload.Length); + var signature = GetSignature(date, payload.Length); HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uriAddress); @@ -84,7 +73,7 @@ public async Task SendOMSApiIngestionFile(string requestBody) if (!(response.StatusCode >= HttpStatusCode.OK && response.StatusCode < HttpStatusCode.Ambiguous)) { m_Tracer.TraceEvent(TraceEventType.Error, 0, "{0} to {1} failed with error {2}", method, address, response.StatusDescription); - throw new HttpException((int)response.StatusCode, string.Format("{0} to {1} failed", method, address)); + throw new HttpException((int)response.StatusCode, $"{method} to {address} failed"); } } @@ -138,31 +127,22 @@ private async Task GetResponse(HttpWebRequest request, Stopwatch s } } - private class RestResponse + private string GetSignature(string date, int contentLength) { - public int SentBytes { get; set; } - - public string StatusDescription { get; set; } - - public string ResponseFromServer { get; set; } + var xHeaders = $"x-ms-date:{date}"; + var stringToHash = $"POST\n{contentLength}\napplication/json\n{xHeaders}\n/api/logs"; - public HttpStatusCode StatusCode { get; set; } - - public TimeSpan Duration { get; set; } + var bytesToHash = Encoding.UTF8.GetBytes(stringToHash); + var keyBytes = Convert.FromBase64String(m_SharedKey); - public override string ToString() + using (var sha256 = new System.Security.Cryptography.HMACSHA256(keyBytes)) { - string escapedResponse = ResponseFromServer.Replace("{", "{{").Replace("}", "}}"); - return string.Format( - CultureInfo.InvariantCulture, - "Sent {0} bytes, status = {1}, status code = {2}, duration = {3}, response = '{4}'", - SentBytes, - StatusDescription, - StatusCode, - Duration, - escapedResponse); + var calculatedHash = sha256.ComputeHash(bytesToHash); + var encodedHash = Convert.ToBase64String(calculatedHash); + + return $"SharedKey {m_CustomerId}:{encodedHash}"; } } } -} +} \ No newline at end of file diff --git a/src/XEL2OMS/Program.cs b/src/XEL2OMS/Program.cs index aeebf61..53e64bd 100644 --- a/src/XEL2OMS/Program.cs +++ b/src/XEL2OMS/Program.cs @@ -1,24 +1,28 @@ -using Microsoft.SqlServer.XEvent.Linq; -using Microsoft.WindowsAzure; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using Microsoft.Practices.EnterpriseLibrary.TransientFaultHandling; +// ----------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +// ----------------------------------------------------------------------- namespace XEL2OMS { - using Microsoft.Azure; - using databaseStateDictionary = Dictionary; - using serverStateDictionary = Dictionary>; - using StateDictionary = Dictionary>>; + using System; + using System.Collections.Generic; + using System.Configuration; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Linq; + using System.Net; + using System.Threading.Tasks; + using Microsoft.Practices.EnterpriseLibrary.TransientFaultHandling; + using Microsoft.SqlServer.XEvent.Linq; + using Microsoft.WindowsAzure.Storage; + using Microsoft.WindowsAzure.Storage.Blob; + using Newtonsoft.Json; + using databaseStateDictionary = System.Collections.Generic.Dictionary; + using serverStateDictionary = System.Collections.Generic.Dictionary>; + using StateDictionary = System.Collections.Generic.Dictionary>>; public static class Program { @@ -85,39 +89,47 @@ private static void PrintHeaders(RequestEventArgs e) } } - private static async Task SendBlobToOMS(CloudPageBlob blob, int eventNumber, OMSIngestionApi oms) + private static async Task SendBlobToOMS(CloudPageBlob blob, int eventNumber) { - RetryPolicy retryPolicy = new RetryPolicy(RetryPolicy.DefaultFixed.ErrorDetectionStrategy, DefaultRetryCount); - - s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Processing: {0}", blob.Uri); - + IEnumerable> chunkedList; + List list; + OmsIngestion service; + OperationContext operationContext; + RetryPolicy retryPolicy; string fileName = Path.Combine(GetLocalStorageFolder(), Path.GetRandomFileName() + ".xel"); + try { - OperationContext operationContext = new OperationContext(); + operationContext = new OperationContext(); + + service = new OmsIngestion( + s_consoleTracer, + ConfigurationManager.AppSettings["omsWorkspaceId"], + ConfigurationManager.AppSettings["omsWorkspaceKey"]); + + retryPolicy = new RetryPolicy(RetryPolicy.DefaultFixed.ErrorDetectionStrategy, DefaultRetryCount); + s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Processing: {0}", blob.Uri); + operationContext.RequestCompleted += (sender, e) => PrintHeaders(e); await retryPolicy.ExecuteAsync((() => blob.DownloadToFileAsync(fileName, FileMode.OpenOrCreate, null, null, operationContext))); - List list; + using (var events = new QueryableXEventData(fileName)) { list = ParseXEL(events, eventNumber, blob.Name); } - IEnumerable> chunkedList = list.Chunk(10000); + + chunkedList = list.Chunk(10000); + foreach (List chunk in chunkedList) { var jsonList = JsonConvert.SerializeObject(chunk); - await oms.SendOMSApiIngestionFile(jsonList); + await service.SendAsync(jsonList); eventNumber += chunk.Count; totalLogs += chunk.Count; } - } - catch (Exception e) - { - s_consoleTracer.TraceEvent(TraceEventType.Error, 0, "Failed processing: {0}. Reason: {1}", blob.Uri, e); - throw; - } - finally - { + + s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Done processing: {0}", blob.Uri); + try { File.Delete(fileName); @@ -126,12 +138,24 @@ private static async Task SendBlobToOMS(CloudPageBlob blob, int eventNumber { s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Was not able to delete file: {0}. Reason: {1}", fileName, e.Message); } + + return eventNumber; + } + catch (Exception e) + { + s_consoleTracer.TraceEvent(TraceEventType.Error, 0, "Failed processing: {0}. Reason: {1}", blob.Uri, e); + throw; + } + finally + { + list = null; + operationContext = null; + retryPolicy = null; + service = null; } - s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Done processing: {0}", blob.Uri); - return eventNumber; } - private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState, OMSIngestionApi oms) + private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState) { int nextEvent = 0; int eventNumber = 0; @@ -185,7 +209,7 @@ private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, database } } - tasks.Add(SendBlobToOMS(blob, eventNumber, oms)); + tasks.Add(SendBlobToOMS(blob, eventNumber)); lastBlob = blobName; lastModified = blob.Properties.LastModified; @@ -212,11 +236,11 @@ private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, database catch (Exception e) { s_consoleTracer.TraceEvent(TraceEventType.Error, 0, "Failed processing sub folder: {0}. Reason: {1}", subfolder.Prefix, e); - UpdateFailuresLog(subfolder.Prefix, e); + auditLogProcessingFailures.Add($"Failed processing audit logs for: {subfolder.Prefix}. Reason: {e.Message}"); } } - private static void SendLogsFromDatabase(CloudBlobDirectory databaseDirectory, serverStateDictionary serverState, OMSIngestionApi oms) + private static void SendLogsFromDatabase(CloudBlobDirectory databaseDirectory, serverStateDictionary serverState) { s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Processing audit logs for database: {0}", databaseDirectory.Prefix); @@ -227,7 +251,7 @@ private static void SendLogsFromDatabase(CloudBlobDirectory databaseDirectory, s foreach (var subfolder in subfolders) { - SendLogsFromSubfolder(subfolder, serverState[databaseName], oms); + SendLogsFromSubfolder(subfolder, serverState[databaseName]); } s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Done processing audit logs for database: {0}", databaseDirectory.Prefix); @@ -235,12 +259,12 @@ private static void SendLogsFromDatabase(CloudBlobDirectory databaseDirectory, s catch (Exception e) { s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Failed processing audit logs for database: {0}. Reason: {1}", databaseDirectory.Prefix, e); - UpdateFailuresLog(databaseDirectory.Prefix, e); + auditLogProcessingFailures.Add($"Failed processing audit logs for: {databaseDirectory.Prefix}. Reason: {e.Message}"); } } - private static void SendLogsFromServer(CloudBlobDirectory serverDirectory, OMSIngestionApi oms) + private static void SendLogsFromServer(CloudBlobDirectory serverDirectory) { s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Processing audit logs for server: {0}", serverDirectory.Prefix); try @@ -250,7 +274,7 @@ private static void SendLogsFromServer(CloudBlobDirectory serverDirectory, OMSIn foreach (var database in databases) { - SendLogsFromDatabase(database, StatesList[serverName], oms); + SendLogsFromDatabase(database, StatesList[serverName]); } s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Done processing audit logs for server: {0}", serverDirectory.Prefix); @@ -258,7 +282,7 @@ private static void SendLogsFromServer(CloudBlobDirectory serverDirectory, OMSIn catch (Exception e) { s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Failed processing audit logs for server: {0}. Reason: {1}", serverDirectory.Prefix, e); - UpdateFailuresLog(serverDirectory.Prefix, e); + auditLogProcessingFailures.Add($"Failed processing audit logs for: {serverDirectory.Prefix}. Reason: {e.Message}"); } } @@ -291,24 +315,15 @@ private static StateDictionary GetStates(string fileName) return statesList; } - private static void UpdateFailuresLog(string resource, Exception ex) - { - string failureMessage = string.Format("Failed processing audit logs for: {0}. Reason: {1}", resource, ex.Message); - auditLogProcessingFailures.Add(failureMessage); - } - static void Main() { - string connectionString = CloudConfigurationManager.GetSetting("ConnectionString"); + string connectionString = ConfigurationManager.AppSettings["ConnectionString"]; string containerName = "sqldbauditlogs"; - string customerId = CloudConfigurationManager.GetSetting("omsWorkspaceId"); - string sharedKey = CloudConfigurationManager.GetSetting("omsWorkspaceKey"); CloudStorageAccount storageAccount; try { - var oms = new OMSIngestionApi(s_consoleTracer, customerId, sharedKey); if (CloudStorageAccount.TryParse(connectionString, out storageAccount) == false) { @@ -323,9 +338,10 @@ static void Main() s_consoleTracer.TraceInformation("Sending logs to OMS"); IEnumerable servers = container.ListBlobs().OfType().ToList(); + foreach (var server in servers) { - SendLogsFromServer(server, oms); + SendLogsFromServer(server); } File.WriteAllText(StateFileName, JsonConvert.SerializeObject(StatesList)); @@ -348,4 +364,4 @@ static void Main() } } } -} +} \ No newline at end of file diff --git a/src/XEL2OMS/RestResponse.cs b/src/XEL2OMS/RestResponse.cs new file mode 100644 index 0000000..cd4d37f --- /dev/null +++ b/src/XEL2OMS/RestResponse.cs @@ -0,0 +1,38 @@ +// ----------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +// ----------------------------------------------------------------------- + +namespace XEL2OMS +{ + using System; + using System.Globalization; + using System.Net; + + public class RestResponse + { + public int SentBytes { get; set; } + + public string StatusDescription { get; set; } + + public string ResponseFromServer { get; set; } + + public HttpStatusCode StatusCode { get; set; } + + public TimeSpan Duration { get; set; } + + public override string ToString() + { + string escapedResponse = ResponseFromServer.Replace("{", "{{").Replace("}", "}}"); + return string.Format( + CultureInfo.InvariantCulture, + "Sent {0} bytes, status = {1}, status code = {2}, duration = {3}, response = '{4}'", + SentBytes, + StatusDescription, + StatusCode, + Duration, + escapedResponse); + } + } +} \ No newline at end of file diff --git a/src/XEL2OMS/SQLAuditLog.cs b/src/XEL2OMS/SQLAuditLog.cs index 1a4824e..cb35175 100644 --- a/src/XEL2OMS/SQLAuditLog.cs +++ b/src/XEL2OMS/SQLAuditLog.cs @@ -1,10 +1,16 @@ -using Microsoft.SqlServer.XEvent.Linq; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; +// ----------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +// ----------------------------------------------------------------------- namespace XEL2OMS { + using System; + using System.Collections.Generic; + using Microsoft.SqlServer.XEvent.Linq; + using Newtonsoft.Json; + public class ClassTypeData { public string ClassTypeDescription { get; private set; } @@ -415,4 +421,4 @@ public SubfolderState() LastModified = DateTimeOffset.MinValue; } } -} +} \ No newline at end of file diff --git a/src/XEL2OMS/XEL2OMS.csproj b/src/XEL2OMS/XEL2OMS.csproj index 91d9da0..260cf46 100644 --- a/src/XEL2OMS/XEL2OMS.csproj +++ b/src/XEL2OMS/XEL2OMS.csproj @@ -56,6 +56,7 @@ prompt MinimumRecommendedRules.ruleset true + true bin\x64\Release\ @@ -68,21 +69,17 @@ true - - packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - True + + packages\Microsoft.Azure.KeyVault.Core.2.0.4\lib\net45\Microsoft.Azure.KeyVault.Core.dll - - packages\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll - True + + packages\Microsoft.Data.Edm.5.8.3\lib\net40\Microsoft.Data.Edm.dll - - packages\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll - True + + packages\Microsoft.Data.OData.5.8.3\lib\net40\Microsoft.Data.OData.dll - - packages\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll - True + + packages\Microsoft.Data.Services.Client.5.8.3\lib\net40\Microsoft.Data.Services.Client.dll packages\EnterpriseLibrary.TransientFaultHandling.6.0.1304.0\lib\portable-net45+win+wp8\Microsoft.Practices.EnterpriseLibrary.TransientFaultHandling.dll @@ -96,23 +93,17 @@ False .\Microsoft.SqlServer.XEvent.Linq.dll - - packages\Microsoft.WindowsAzure.ConfigurationManager.3.2.1\lib\net40\Microsoft.WindowsAzure.Configuration.dll - True + + packages\WindowsAzure.Storage.8.6.0\lib\net45\Microsoft.WindowsAzure.Storage.dll - - packages\WindowsAzure.Storage.7.2.1\lib\net40\Microsoft.WindowsAzure.Storage.dll - True - - - packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll - True + + packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll + - - packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll - True + + packages\System.Spatial.5.8.3\lib\net40\System.Spatial.dll @@ -123,16 +114,19 @@ - + + Designer - + + Designer + diff --git a/src/XEL2OMS/packages.config b/src/XEL2OMS/packages.config index a13b089..b75f6e4 100644 --- a/src/XEL2OMS/packages.config +++ b/src/XEL2OMS/packages.config @@ -1,12 +1,11 @@  - - - - - - - - + + + + + + + \ No newline at end of file