Merge pull request #85 from cloudgraphdev/beta

Release 0.67.0

Author: tyler-dunkel

CHANGELOG.md

@@ -1,3 +1,79 @@
+# [0.67.0-beta.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-beta.1) (2023-07-19)
+
+
+### Bug Fixes
+
+* Added missing attribute for cosmosdb schema ([cde0c79](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/cde0c791374d648f77541b1189ddbd3062ea67aa))
+
+
+### Features
+
+* add publicIp missing services ([33ae7da](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/33ae7da24dd69264344016bf4af3b25a6264c43d))
+* Added missing subnets ids and endpoint policies for subnets ([baa0a6e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/baa0a6e72510f70e217a5c516479db8d5d05294f))
+* **azure:** Add missing dataFactory services ([2b9a48b](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/2b9a48b9e848aad2775069167dcb406ad0dc8f77))
+* **azure:** add missing mySQL Server services ([90131c8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/90131c83d231a8ef3d2f29b23412e978688e32dc))
+* **azure:** Add missing postgreSql server services ([8aeff96](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8aeff969f44cfa8751fefacbbca6597cf7213cd1))
+* **azure:** Add missing SQL Server services ([41bffd4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/41bffd48c728303d0d697509c9efa9a5c7442166))
+* **azure:** add route table services ([5573e8e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/5573e8eeee9387d50e440e7d1aecd4c529f3c531))
+* **azure:** Support missing storageBlob and storageContainer services ([b179ebf](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/b179ebf76107a6285c978067b79f544e995df66f))
+* collect crawl error logs ([bd58d1d](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/bd58d1dce0cd55d86828b0a463d41061352f2517))
+* Exposed containersIds for cosmosdb account ([8b45320](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8b4532048fc8137539489adb38eee101ff867039))
+* Fetched records for private dns ([83cbdd8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/83cbdd809f81e750d5abc83bf503dba47e12fe50))
+* Fetched virtual network links for private zone ([414ceec](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/414ceece3af505c18a76d6141d63a8d85e20067b))
+
+# [0.67.0-alpha.6](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.5...0.67.0-alpha.6) (2023-07-19)
+
+
+### Features
+
+* collect crawl error logs ([bd58d1d](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/bd58d1dce0cd55d86828b0a463d41061352f2517))
+
+# [0.67.0-alpha.5](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.4...0.67.0-alpha.5) (2023-06-14)
+
+
+### Features
+
+* add publicIp missing services ([33ae7da](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/33ae7da24dd69264344016bf4af3b25a6264c43d))
+* Added missing subnets ids and endpoint policies for subnets ([baa0a6e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/baa0a6e72510f70e217a5c516479db8d5d05294f))
+* **azure:** Add missing dataFactory services ([2b9a48b](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/2b9a48b9e848aad2775069167dcb406ad0dc8f77))
+* **azure:** add route table services ([5573e8e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/5573e8eeee9387d50e440e7d1aecd4c529f3c531))
+* Fetched records for private dns ([83cbdd8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/83cbdd809f81e750d5abc83bf503dba47e12fe50))
+* Fetched virtual network links for private zone ([414ceec](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/414ceece3af505c18a76d6141d63a8d85e20067b))
+
+# [0.67.0-alpha.4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.3...0.67.0-alpha.4) (2023-06-09)
+
+
+### Features
+
+* **azure:** add missing mySQL Server services ([90131c8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/90131c83d231a8ef3d2f29b23412e978688e32dc))
+
+# [0.67.0-alpha.3](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.2...0.67.0-alpha.3) (2023-06-07)
+
+
+### Features
+
+* **azure:** Add missing postgreSql server services ([8aeff96](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8aeff969f44cfa8751fefacbbca6597cf7213cd1))
+
+# [0.67.0-alpha.2](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.1...0.67.0-alpha.2) (2023-06-06)
+
+
+### Bug Fixes
+
+* Added missing attribute for cosmosdb schema ([cde0c79](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/cde0c791374d648f77541b1189ddbd3062ea67aa))
+
+
+### Features
+
+* Exposed containersIds for cosmosdb account ([8b45320](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8b4532048fc8137539489adb38eee101ff867039))
+
+# [0.67.0-alpha.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-alpha.1) (2023-06-06)
+
+
+### Features
+
+* **azure:** Add missing SQL Server services ([41bffd4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/41bffd48c728303d0d697509c9efa9a5c7442166))
+* **azure:** Support missing storageBlob and storageContainer services ([b179ebf](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/b179ebf76107a6285c978067b79f544e995df66f))
+
 # [0.66.0](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.65.0...0.66.0) (2023-05-16)
 
 

README.md

@@ -4,10 +4,13 @@ Scan cloud infrastructure via the [Azure SDK](https://github.com/Azure/azure-sdk
 
 <!-- toc -->
 
-- [Install](#install)
-- [Authentication](#authentication)
-- [Supported Services](#supported-services)
-<!-- tocstop -->
+- [CloudGraph Azure Provider](#cloudgraph-azure-provider)
+  - [Docs](#docs)
+  - [Install](#install)
+  - [Authentication](#authentication)
+  - [Supported Services](#supported-services)
+  - [Development](#development)
+  - [Testing](#testing)
 
 ## Docs
 
@@ -95,6 +98,7 @@ CloudGraph needs read permissions in order to ingest your data. To keep things e
 | postgreSqlServers                           | resourceGroup, databasePostgreSql                                             |
 | privateDns                                  | resourceGroup                                                                 |
 | publicIp                                    | networkInterface, resourceGroup                                               |
+| publicIpPrefix                              |                                                                               |
 | recoveryInstances                           | recoveryVaults, resourceGroup                                                 |
 | recoveryPolicies                            | recoveryVaults, resourceGroup                                                 |
 | recoveryVaults                              | recoveryInstances, recoveryPolicies, resourceGroup                            |
@@ -104,6 +108,8 @@ CloudGraph needs read permissions in order to ingest your data. To keep things e
 | replicationNetworks                         | resourceGroup                                                                 |
 | replicationPolicies                         | resourceGroup                                                                 |
 | resourceGroup                               | **all services**                                                              |
+| routeFilter                                 |                                                                               |
+| routeTable                                  |                                                                               |
 | securityAssessments                         |                                                                               |
 | securityContacts                            |                                                                               |
 | securityGroup                               | networkInterface, resourceGroup                                               |

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloudgraph/cg-provider-azure",
-  "version": "0.66.0",
+  "version": "0.67.0-beta.1",
   "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
@@ -73,7 +73,7 @@
     "@azure/identity": "^2.0.4",
     "@azure/storage-blob": "^12.8.0",
     "@azure/storage-queue": "^12.9.0",
-    "@cloudgraph/sdk": "0.22.1",
+    "@cloudgraph/sdk": "0.23.0",
     "@graphql-tools/load-files": "^6.5.3",
     "@graphql-tools/merge": "^8.2.3",
     "@microsoft/microsoft-graph-client": "^3.0.2",

src/enums/schemasMap.ts

@@ -77,6 +77,8 @@ export default {
   [services.replicationNetworks]: 'azureReplicationNetwork',
   [services.replicationPolicies]: 'azureReplicationPolicy',
   [services.resourceGroup]: 'azureResourceGroup',
+  [services.routeFilter]: 'azureRouteFilter',
+  [services.routeTable]: 'azureRouteTable',
   [services.securityAssesments]: 'azureSecurityAssesment',
   [services.securityContacts]: 'azureSecurityContact',
   [services.securityGroup]: 'azureNetworkSecurityGroup',

src/enums/serviceAliases.ts

@@ -56,6 +56,7 @@ export default {
   [services.postgreSqlServers]: 'postgreSqlServers',
   [services.privateDns]: 'privateDnsZones',
   [services.publicIp]: 'publicIps',
+  [services.publicIpPrefix]: 'publicIpPrefixes',
   [services.recoveryVaults]: 'recoveryVaults',
   [services.recoveryInstances]: 'recoveryInstances',
   [services.recoveryPolicies]: 'recoveryPolicies',
@@ -65,6 +66,8 @@ export default {
   [services.replicationNetworks]: 'replicationNetworks',
   [services.replicationPolicies]: 'replicationPolicies',
   [services.resourceGroup]: 'resourceGroups',
+  [services.routeFilter]: 'routeFilters',
+  [services.routeTable]: 'routeTables',
   [services.securityAssesments]: 'securityAssesments',
   [services.securityContacts]: 'securityContacts',
   [services.securityGroup]: 'securityGroups',

src/enums/serviceMap.ts

@@ -51,6 +51,7 @@ import AzurePolicyAssignment from '../services/policyAssignment'
 import AzurePostgreSqlServer from '../services/postgreSqlServers'
 import AzurePrivateDns from '../services/privateDns'
 import AzurePublicIp from '../services/publicIp'
+import AzurePublicIpPrefix from '../services/publicIpPrefix'
 import AzureRecoveryVault from '../services/recoveryVaults'
 import AzureRecoveryInstance from '../services/recoveryInstances'
 import AzureRecoveryPolicy from '../services/recoveryPolicies'
@@ -89,6 +90,8 @@ import AzureBilling from '../services/billing'
 import AzureLogProfiles from '../services/logProfiles'
 import Subscription from '../services/subscription'
 import AzureNetworkWatcher from '../services/networkWatcher'
+import AzureRouteTable from '../services/routeTable'
+import AzureRouteFilter from '../services/routeFilter'
 
 /**
  * serviceMap is an object that contains all currently supported services for AZURE
@@ -160,6 +163,7 @@ export default {
   [services.postgreSqlServers]: AzurePostgreSqlServer,
   [services.privateDns]: AzurePrivateDns,
   [services.publicIp]: AzurePublicIp,
+  [services.publicIpPrefix]: AzurePublicIpPrefix,
   [services.recoveryVaults]: AzureRecoveryVault,
   [services.recoveryInstances]: AzureRecoveryInstance,
   [services.recoveryPolicies]: AzureRecoveryPolicy,
@@ -169,6 +173,8 @@ export default {
   [services.replicationNetworks]: AzureReplicationNetwork,
   [services.replicationPolicies]: AzureReplicationPolicy,
   [services.resourceGroup]: AzureResourceGroup,
+  [services.routeFilter]: AzureRouteFilter,
+  [services.routeTable]: AzureRouteTable,
   [services.securityAssesments]: AzureSecurityAssesments,
   [services.securityContacts]: AzureSecurityContacts,
   [services.securityGroup]: AzureNetworkSecurityGroup,

src/enums/services.ts

@@ -62,6 +62,7 @@ export default {
   postgreSqlServers: 'postgreSqlServers',
   privateDns: 'privateDns',
   publicIp: 'publicIp',
+  publicIpPrefix: 'publicIpPrefix',
   recoveryVaults: 'recoveryVaults',
   recoveryInstances: 'recoveryInstances',
   recoveryPolicies: 'recoveryPolicies',
@@ -71,6 +72,8 @@ export default {
   replicationNetworks: 'replicationNetworks',
   replicationPolicies: 'replicationPolicies',
   resourceGroup: 'resourceGroup',
+  routeFilter: 'routeFilter',
+  routeTable: 'routeTable',
   securityAssesments: 'securityAssesments',
   securityContacts: 'securityContacts',
   securityGroup: 'securityGroup',

src/properties/logger.ts

@@ -20,7 +20,8 @@ export default {
   foundAKSManagedClusters: (num: number): string =>
     `Found ${num} AKS managed clusters`,
   /* App Gateway */
-  foundApplicationGateway: (num: number): string => `Found ${num} Application Gateways`,
+  foundApplicationGateway: (num: number): string =>
+    `Found ${num} Application Gateways`,
   /* App Service */
   foundAppServiceEnvironments: (num: number): string =>
     `Found ${num} app service environments`,
@@ -152,11 +153,14 @@ export default {
     `Found ${num} private dns zones`,
   // Public Ips
   foundPublicIps: (num: number): string => `Found ${num} public ips`,
+  foundPublicIpPrefixes: (num: number): string =>
+    `Found ${num} public ip prefixes`,
   // Recovery Vaults
   foundRecoveryVaults: (num: number): string => `Found ${num} recovery vaults`,
   foundRecoveryInstances: (num: number): string =>
-  `Found ${num} recovery instances`,
-  foundRecoveryPolicies: (num: number): string => `Found ${num} recovery policies`,
+    `Found ${num} recovery instances`,
+  foundRecoveryPolicies: (num: number): string =>
+    `Found ${num} recovery policies`,
   // RedisCache
   foundRedisCaches: (num: number): string => `Found ${num} Redis caches`,
   // Replication Appliances
@@ -173,6 +177,10 @@ export default {
     `Found ${num} replication policies`,
   // Resource Groups
   foundResourceGroups: (num: number): string => `Found ${num} resource groups`,
+  // Route Filters
+  foundRouteFilters: (num: number): string => `Found ${num} route filters`,
+  // Route Tables
+  foundRouteTables: (num: number): string => `Found ${num} route tables`,
   // Security Assesments
   foundSecurityAssesments: (num: number): string =>
     `Found ${num} security assesments`,

src/services/cosmosDb/format.ts

@@ -31,9 +31,11 @@ const formatRestoreParameters = (
   const { restoreTimestampInUtc, databasesToRestore, ...rest } = restoreParameters
   return {
     restoreTimestampInUtc: restoreTimestampInUtc?.toISOString(),
-    databasesToRestore: databasesToRestore?.map(db => ({ id: generateUniqueId({
-      ...db
-    }), ...db })) || [],
+    databasesToRestore: databasesToRestore?.map(db => ({
+      id: generateUniqueId({
+        ...db
+      }), ...db
+    })) || [],
     ...rest,
   }
 }
@@ -94,6 +96,20 @@ export default ({
     tables = [],
   } = service
 
+  let containersIds: string[] = []
+  const sqlDatabases = databases?.map(({ id: databaseId, options, data, ...rest }) => {
+    const containers = new Set(data.map(d => d.id))
+    containersIds = Array.from(containers)
+    return ({
+      id: databaseId,
+      ...rest,
+      options: {
+        throughput: options?.throughput,
+        maxThroughput: options?.autoscaleSettings?.maxThroughput,
+      },
+    })
+  }) || []
+
   return {
     id,
     name,
@@ -133,10 +149,10 @@ export default ({
       failoverPolicies?.map(fp => ({ id: fp.id, ...fp })) || [],
     virtualNetworkRules:
       virtualNetworkRules?.map(vn => ({ id: vn.id, ...vn })) || [],
-    privateEndpointConnections: privateEndpointConnections?.map(({ id: endpointId, privateEndpoint, ...pe}) => ({ 
-      id: endpointId, 
+    privateEndpointConnections: privateEndpointConnections?.map(({ id: endpointId, privateEndpoint, ...pe }) => ({
+      id: endpointId,
       privateEndpointId: privateEndpoint?.id,
-      ...pe 
+      ...pe
     })) || [],
     enableMultipleWriteLocations,
     enableCassandraConnector,
@@ -160,14 +176,8 @@ export default ({
     disableLocalAuth,
     capacityTotalThroughputLimit: capacity?.totalThroughputLimit,
     tags: formatTagsFromMap(Tags),
-    databases: databases?.map(({ id: databaseId, options, ...rest }) => ({
-      id: databaseId,
-      ...rest,
-      options: {
-        throughput: options?.throughput,
-        maxThroughput: options?.autoscaleSettings?.maxThroughput,
-      },
-    })) || [],
+    databases: sqlDatabases,
     azureTables: tables?.map(at => ({ id: at.id, ...at })) || [],
+    containersIds
   }
 }

src/services/cosmosDb/schema.graphql

@@ -303,4 +303,5 @@ type azureCosmosDb implements azureResource
   databases: [azureCosmosDbDatabase]
   azureTables: [azureCosmosDbTable]
   resourceGroup: [azureResourceGroup] @hasInverse(field: cosmosDb)
+  containersIds: [String] @search(by: [hash])
 }