Added retries of failed jobs in CI e2e checks (#19296)

* Draft update

* add build

* Rm deprecated axios types

* Refactor: Moved helpers to separate files + add common config

* Make config readonly

* Fix _generated dir check

* fix rerun call

* Add content type

* Add "canceling" state

* Fix get of build configs

* Update logs

* Update pipeline run steps

* Add cache for fetch pipelines

Author: localconst

ci/ci-test-tasks/canary-tests-v2.yml

@@ -28,13 +28,21 @@ jobs:
       - name: tasks
         value: $[dependencies.detect_changes.outputs['detect.TASKS']]
   pool:
-    vmImage: ubuntu-20.04
+    vmImage: ubuntu-latest
   steps:
+  - task: NodeTool@0
+    displayName: Use Node 20.x
+    inputs:
+      versionSpec: 20.x
+
   - script: |
-      npm i axios
-    displayName: npm i axios
+      cd ./ci/ci-test-tasks/test-and-verify-v2
+      npm i
+      npm run build
+    displayName: 'Build test pipelines runner'
+
   - script: |
-      echo $(tasks)
-      node ./ci/ci-test-tasks/test-and-verify-v2.js $(ADOToken) $(ADOUrl) $(System.TeamProject) $(tasks)
+      echo "Tasks to test: $(tasks)"
+      node ./ci/ci-test-tasks/test-and-verify-v2 $(ADOToken) $(ADOUrl) $(System.TeamProject) $(tasks)
     displayName: 'Run test pipelines and verify results'
     failOnStderr: true

ci/ci-test-tasks/schedules.yml

@@ -34,11 +34,19 @@ jobs:
   pool:
     vmImage: ubuntu-latest
   steps:
+  - task: NodeTool@0
+    displayName: Use Node 20.x
+    inputs:
+      versionSpec: 20.x
+
+  - script: |
+      cd ./ci/ci-test-tasks/test-and-verify-v2
+      npm i
+      npm run build
+    displayName: 'Build test pipelines runner'
+
   - bash: |
-      npm i axios
-    displayName: npm i axios
-  - bash: |
-      echo $(tasks)
-      node ./ci/ci-test-tasks/test-and-verify-v2.js $(ADOToken) $(ADOUrl) $(System.TeamProject) $(tasks)
+      echo "Tasks to test: $(tasks)"
+      node ./ci/ci-test-tasks/test-and-verify-v2 $(ADOToken) $(ADOUrl) $(System.TeamProject) $(tasks)
     displayName: Run test pipelines and verify results
     failOnStderr: true

ci/ci-test-tasks/test-and-verify-v2.js

@@ -0,0 +1 @@
+dist

ci/ci-test-tasks/test-and-verify-v2/.gitignore

@@ -0,0 +1 @@
+require(__dirname + '/dist/index.js')