feat: collect crawl error logs

Author: aacotroneo

package.json

@@ -73,7 +73,7 @@
     "@azure/identity": "^2.0.4",
     "@azure/storage-blob": "^12.8.0",
     "@azure/storage-queue": "^12.9.0",
-    "@cloudgraph/sdk": "0.22.1",
+    "@cloudgraph/sdk": "0.23.0",
     "@graphql-tools/load-files": "^6.5.3",
     "@graphql-tools/merge": "^8.2.3",
     "@microsoft/microsoft-graph-client": "^3.0.2",
@@ -126,4 +126,4 @@
     "**/glob-parent": "^5.1.2",
     "**/uri-js": "^3.0.1"
   }
-}
+}

src/services/index.ts

@@ -25,7 +25,7 @@ import {
   GLOBAL_REGION,
 } from '../config/constants'
 import { obfuscateSensitiveString } from '../utils/format'
-import { checkAndMergeConnections, sortResourcesDependencies } from '../utils'
+import { getAllProviderErrors, sortResourcesDependencies } from '../utils'
 import { createDiffSecs } from '../utils/dateutils'
 import {
   getClientSecretCredentials,
@@ -151,9 +151,10 @@ export default class Provider extends CloudGraph.Client {
       )}`
     )
     this.logger.success(
-      `subscriptionIds: ${this.subscriptions.length > 1
-        ? this.subscriptions.join(', ')
-        : this.subscriptions[0]
+      `subscriptionIds: ${
+        this.subscriptions.length > 1
+          ? this.subscriptions.join(', ')
+          : this.subscriptions[0]
       }`
     )
     this.logger.success(
@@ -700,11 +701,15 @@ export default class Provider extends CloudGraph.Client {
       }
     }
 
-    return this.enhanceData({
+    const enhancedData = await this.enhanceData({
       subscriptions: subscriptions.data[GLOBAL_REGION],
       configuredRegions,
       rawData,
       data: result,
     })
+
+    const errors = getAllProviderErrors()
+
+    return { ...enhancedData, errors }
   }
 }

src/utils/index.ts

@@ -3,6 +3,7 @@ import camelCase from 'lodash/camelCase'
 import isEmpty from 'lodash/isEmpty'
 import unionWith from 'lodash/unionWith'
 import isEqual from 'lodash/isEqual'
+import { ProviderError } from '@cloudgraph/sdk/dist/src/types'
 import relations from '../enums/relations'
 import {
   AzureDebugScope,
@@ -65,6 +66,12 @@ export function initTestConfig(): void {
   jest.setTimeout(900000)
 }
 
+const errorsHistory: ProviderError[] = []
+
+export function getAllProviderErrors(): ProviderError[] {
+  return errorsHistory
+}
+
 export function generateAzureErrorLog(
   service: string,
   functionName: string,
@@ -87,6 +94,13 @@ export function generateAzureErrorLog(
   } else {
     logger.debug(err.message)
   }
+  if (err?.statusCode !== 429) {
+    errorsHistory.push({
+      service,
+      function: functionName,
+      message: err?.message || 'Unknown error',
+    })
+  }
   throw new Error()
 }
 

yarn.lock

@@ -1282,10 +1282,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@cloudgraph/sdk@0.22.1":
-  version "0.22.1"
-  resolved "https://registry.yarnpkg.com/@cloudgraph/sdk/-/sdk-0.22.1.tgz#bf5c2f263df17b085147a3ca234ac5b0a8806545"
-  integrity sha512-aMrnoTRRIODAVvgtmqGZPB7Z8x1kMgKjFtLjpiHa121o0GCEZCjbxDmU8T3DwmRwI1u4QD5iKbXf9dB0573mJA==
+"@cloudgraph/sdk@0.23.0":
+  version "0.23.0"
+  resolved "https://registry.yarnpkg.com/@cloudgraph/sdk/-/sdk-0.23.0.tgz#3f6012c0e3573f527da27ebcccb8841bbeffd21b"
+  integrity sha512-c+Ozu8yLK3RlAcKy4d0fdTQc9JkdUJ2ESmrcW7oC5XdIARrDWQ5+Kr8AGPABOK1/UYtFyVR2JmUHnoSJdGAv6Q==
   dependencies:
     "@graphql-tools/load-files" "^6.5.3"
     "@graphql-tools/merge" "^8.2.1"