Merge pull request #12295 from Azure/v-pmalreddy-cdn

Google Cloud Platform CDN CCF Connector - Standard Table

Author: v-atulyadav

Solutions/GoogleCloudPlatformCDN/Data Connectors/GCPCDNLogs_ccp/GCPCDNLogs_ConnectorDefinition.json

@@ -7,10 +7,10 @@
   "properties": {
     "connectorUiConfig": {
       "id": "GCPCDNLogsCCPDefinition",
-      "title": "Google Cloud Platform CDN",
+      "title": "Google Cloud Platform CDN (via Codeless Connector Framework) (Preview)",
       "publisher": "Microsoft",
       "descriptionMarkdown": "The Google Cloud Platform CDN data connector provides the capability to ingest Cloud CDN Audit logs and Cloud CDN Traffic logs into Microsoft Sentinel using the Compute Engine API. Refer the [Product overview](https://cloud.google.com/cdn/docs/overview) document for more details.",
-      "graphQueriesTableName": "GCP_CDNV2_CL",
+      "graphQueriesTableName": "GCPCDN",
       "graphQueries": [
         {
           "metricName": "Total events received",

Solutions/GoogleCloudPlatformCDN/Data Connectors/GCPCDNLogs_ccp/GCPCDNLogs_DCR.json

@@ -6,68 +6,6 @@
         "location": "{{location}}",
         "properties": {
             "dataCollectionEndpointId": "{{dataCollectionEndpointId}}",
-            "streamDeclarations": {
-                "Custom-GCPCDN": {
-                    "columns": [
-                        {
-                            "name": "insertId",
-                            "type": "string"
-                        },
-                        {
-                            "name": "logName",
-                            "type": "string"
-                        },
-                        {
-                            "name": "protoPayload",
-                            "type": "dynamic"
-                        },
-                        {
-                            "name": "resource",
-                            "type": "dynamic"
-                        },
-                        {
-                            "name": "severity",
-                            "type": "string"
-                        },
-                        {
-                            "name": "timestamp",
-                            "type": "datetime"
-                        },
-                        {
-                            "name": "jsonPayload",
-                            "type": "dynamic"
-                        },
-                        {
-                            "name": "labels",
-                            "type": "dynamic"
-                        },
-                        {
-                            "name": "receiveTimestamp",
-                            "type": "datetime"
-                        },
-                        {
-                            "name": "operation",
-                            "type": "dynamic"
-                        },
-                        {
-                            "name": "spanId",
-                            "type": "string"
-                        },
-                        {
-                            "name": "trace",
-                            "type": "string"
-                        },
-                        {
-                            "name": "httpRequest",
-                            "type": "dynamic"
-                        },
-                        {
-                            "name": "TimeGenerated",
-                            "type": "datetime"
-                        }
-                    ]
-                }
-            },
             "destinations": {
                 "logAnalytics": [
                     {
@@ -79,13 +17,11 @@
             "dataFlows": [
                 {
                     "streams": [
-                        "Custom-GCPCDN"
+                        "Microsoft-GCPCDN"
                     ],
                     "destinations": [
                         "clv2ws1"
-                    ],
-                    "transformKql": "source | extend ProtoPayload = parse_json(protoPayload) | extend JsonPayload = parse_json(jsonPayload) | extend Operation = parse_json(operation) | extend Labels = parse_json(labels) | extend Resource = parse_json(resource) | extend HttpRequest = parse_json(httpRequest) | extend TimeGenerated = now(), InsertID = insertId, Timestamp = timestamp, Severity = severity, LogName = logName, ReceiveTimestamp = receiveTimestamp, SpanID = spanId, Trace = trace, LabelsAPIRootTriggerID = tostring(Labels['compute.googleapis.com/root_trigger_id']), ResourceType = tostring(Resource['type']), ResourceLabelsBackendServiceID = tostring(Resource['labels']['backend_service_id']), ResourceLabelsBackendBucketID = tostring(Resource['labels']['backend_bucket_id']), ResourceLabelsProjectID = tostring(Resource['labels']['project_id']), ResourceLabelsLocation = tostring(Resource['labels']['location']), PayloadType = tostring(ProtoPayload['@type']), MethodName = tostring(ProtoPayload['methodName']), NumResponseItems = tostring(ProtoPayload['numResponseItems']), ResourceName = tostring(ProtoPayload['resourceName']), ServiceName = tostring(ProtoPayload['serviceName']), AuthenticationInfoPrincipalEmail = tostring(ProtoPayload['authenticationInfo']['principalEmail']), AuthorizationInfo = tostring(ProtoPayload['authorizationInfo']), RequestType = tostring(ProtoPayload['request']['@type']), RequestKeyName = tostring(ProtoPayload['request']['keyName']), RequestGroup = tostring(ProtoPayload['request']['group']), RequestSecurityPolicy = tostring(ProtoPayload['request']['securityPolicy']), RequestCompressionMode = tostring(ProtoPayload['request']['compressionMode']), RequestDescription = tostring(ProtoPayload['request']['description']), RequestEnableCDN = tobool(ProtoPayload['request']['enableCDN']), RequestIPAddressSelectionPolicy = tostring(ProtoPayload['request']['ipAddressSelectionPolicy']), RequestLoadBalancingScheme = tostring(ProtoPayload['request']['loadBalancingScheme']), RequestLocalityLbPolicy = tostring(ProtoPayload['request']['localityLbPolicy']), RequestName = tostring(ProtoPayload['request']['name']), RequestPortName = tostring(ProtoPayload['request']['portName']), RequestProtocol = tostring(ProtoPayload['request']['protocol']), RequestSessionAffinity = tostring(ProtoPayload['request']['sessionAffinity']), RequestTimeoutSec = tostring(ProtoPayload['request']['timeoutSec']), RequestBackends = tostring(ProtoPayload['request']['backends']), RequestCDNPolicySignedUrlCacheMaxAgeSec = tostring(ProtoPayload['request']['cdnPolicy']['signedUrlCacheMaxAgeSec']), RequestFingerprint = tostring(ProtoPayload['request']['fingerprint']), RequestCDNPolicyCacheMode = tostring(ProtoPayload['request']['cdnPolicy']['cacheMode']), RequestCDNPolicyClientTtl = tostring(ProtoPayload['request']['cdnPolicy']['clientTtl']), RequestCDNPolicyDefaultTtl = tostring(ProtoPayload['request']['cdnPolicy']['defaultTtl']), RequestCDNPolicyMaxTtl = tostring(ProtoPayload['request']['cdnPolicy']['maxTtl']), RequestCDNPolicyNegativeCaching = tobool(ProtoPayload['request']['cdnPolicy']['negativeCaching']), RequestCDNPolicyServeWhileStale = tostring(ProtoPayload['request']['cdnPolicy']['serveWhileStale']), RequestCDNPolicyCacheKeyPolicyIncludeHost = tobool(ProtoPayload['request']['cdnPolicy']['cacheKeyPolicy']['includeHost']), RequestCDNPolicyCacheKeyPolicyIncludeProtocol = tobool(ProtoPayload['request']['cdnPolicy']['cacheKeyPolicy']['includeProtocol']), RequestCDNPolicyCacheKeyPolicyIncludeQueryString = tobool(ProtoPayload['request']['cdnPolicy']['cacheKeyPolicy']['includeQueryString']), RequestConnectionDrainingTimeoutSec = tostring(ProtoPayload['request']['connectionDraining']['drainingTimeoutSec']), RequestHealthChecks = tostring(ProtoPayload['request']['healthChecks']), RequestLogConfigEnable = tobool(ProtoPayload['request']['logConfig']['enable']), RequestLogConfigSampleRate = tostring(ProtoPayload['request']['logConfig']['sampleRate']), RequestMetadataCallerIP = tostring(ProtoPayload['requestMetadata']['callerIp']), RequestMetadataCallerSuppliedUserAgent = tostring(ProtoPayload['requestMetadata']['callerSuppliedUserAgent']), RequestMetadataDestinationAttributes = tostring(ProtoPayload['requestMetadata']['destinationAttributes']), RequestMetadataRequestAttributesAuth = tostring(ProtoPayload['requestMetadata']['requestAttributes']['auth']), RequestMetadataRequestAttributesTime = todatetime(ProtoPayload['requestMetadata']['requestAttributes']['time']), RequestMetadataRequestAttributesReason = tostring(ProtoPayload['requestMetadata']['requestAttributes']['reason']), ResourceLocationCurrentLocations = tostring(ProtoPayload['resourceLocation']['currentLocations']), ResponseType = tostring(ProtoPayload['response']['@type']), ResponseID = tostring(ProtoPayload['response']['id']), ResponseInsertTime = todatetime(ProtoPayload['response']['insertTime']), ResponseName = tostring(ProtoPayload['response']['name']), ResponseOperationType = tostring(ProtoPayload['response']['operationType']), ResponseProgress = tostring(ProtoPayload['response']['progress']), ResponseSelfLink = tostring(ProtoPayload['response']['selfLink']), ResponseSelfLinkWithID = tostring(ProtoPayload['response']['selfLinkWithId']), ResponseStartTime = todatetime(ProtoPayload['response']['startTime']), ResponseStatus = tostring(ProtoPayload['response']['status']), ResponseTargetID = tostring(ProtoPayload['response']['targetId']), ResponseTargetLink = tostring(ProtoPayload['response']['targetLink']), ResponseUser = tostring(ProtoPayload['response']['user']), OperationFirst = tobool(Operation['first']), OperationLast = tobool(Operation['last']), OperationID = tostring(Operation['id']), OperationProducer = tostring(Operation['producer']), HttpRequestCacheLookup = tobool(HttpRequest['cacheLookup']), HttpRequestLatency = tostring(HttpRequest['latency']), HttpRequestRemoteIP = tostring(HttpRequest['remoteIp']), HttpRequestRequestMethod = tostring(HttpRequest['requestMethod']), HttpRequestRequestSize = tostring(HttpRequest['requestSize']), HttpRequestRequestUrl = tostring(HttpRequest['requestUrl']), HttpRequestResponseSize = tostring(HttpRequest['responseSize']), HttpRequestCacheFillBytes = tostring(HttpRequest['cacheFillBytes']), HttpRequestCacheHit = tobool(HttpRequest['cacheHit']), HttpRequestServerIP = tostring(HttpRequest['serverIp']), HttpRequestStatus = todouble(HttpRequest['status']), HttpRequestUserAgent = tostring(HttpRequest['userAgent']), JsonPayloadType = tostring(JsonPayload['@type']), BackendTargetProjectNumber = tostring(JsonPayload['backendTargetProjectNumber']), CacheDecision = tostring(JsonPayload['cacheDecision']), CacheID = tostring(JsonPayload['cacheId']), RemoteIP = tostring(JsonPayload['remoteIp']), StatusDetails = tostring(JsonPayload['statusDetails']), EnforcedEdgeSecurityPolicyConfiguredAction = tostring(JsonPayload['enforcedEdgeSecurityPolicy']['configuredAction']), EnforcedEdgeSecurityPolicyName = tostring(JsonPayload['enforcedEdgeSecurityPolicy']['name']), EnforcedEdgeSecurityPolicyOutcome = tostring(JsonPayload['enforcedEdgeSecurityPolicy']['outcome']), EnforcedEdgeSecurityPolicyPriority = tostring(JsonPayload['enforcedEdgeSecurityPolicy']['priority']), OverrideResponseCode = todouble(JsonPayload['overrideResponseCode']), ErrorService = tostring(JsonPayload['errorService']), ErrorBackendStatusDetails = tostring(JsonPayload['errorBackendStatusDetails']), AuthzPolicyInfoPolicies = tostring(JsonPayload['authzPolicyInfo']['policies']), AuthzPolicyInfoResult = tostring(JsonPayload['authzPolicyInfo']['result']), LoadBalancingScheme = tostring(JsonPayload['loadBalancingScheme']), ResourceLabelsBackendServiceName = tostring(Resource['labels']['backend_service_name']), ResourceLabelsForwardingRuleName = tostring(Resource['labels']['forwarding_rule_name']), ResourceLabelsTargetProxyName = tostring(Resource['labels']['target_proxy_name']), ResourceLabelsUrlMapName = tostring(Resource['labels']['url_map_name']), ResourceLabelsZone = tostring(Resource['labels']['zone']) | project-away protoPayload, resource, jsonPayload, labels, operation, httpRequest",
-                    "outputStream": "Custom-GCP_CDNV2_CL"
+                    ]
                 }
             ]
         }

Solutions/GoogleCloudPlatformCDN/Data Connectors/GCPCDNLogs_ccp/GCPCDNLogs_PollingConfig.json

@@ -7,9 +7,9 @@
         "location": "{{location}}",
         "properties": {
             "connectorDefinitionName": "GCPCDNLogsCCPDefinition",
-            "dataType": "GCP_CDNV2_CL",
+            "dataType": "GCPCDN",
             "dcrConfig": {
-                "streamName": "Custom-GCPCDN",
+                "streamName": "SENTINEL_GCP_CDN_LOGS",
                 "dataCollectionEndpoint": "{{dataCollectionEndpoint}}",
                 "dataCollectionRuleImmutableId": "{{dataCollectionRuleImmutableId}}"
             },