Merge pull request #618 from phani-srikar/cb

phani-srikar · web-flow · commit ab66222da3ea · 2023-07-12T16:32:58.000-07:00
Split E2E tests into batches
diff --git a/.circleci/config.base.yml b/.circleci/config.base.yml
@@ -303,7 +303,7 @@ jobs:
             yarn clean-e2e-resources
           no_output_timeout: 20m
       - store_artifacts:
-          path: ~/repo/packages/amplify-e2e-tests/amplify-e2e-reports
+          path: ~/repo/packages/amplify-codegen-e2e-tests/amplify-e2e-reports
     working_directory: ~/repo
 
   verify-api-extract:
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -306,7 +306,7 @@ jobs:
             yarn clean-e2e-resources
           no_output_timeout: 20m
       - store_artifacts:
-          path: ~/repo/packages/amplify-e2e-tests/amplify-e2e-reports
+          path: ~/repo/packages/amplify-codegen-e2e-tests/amplify-e2e-reports
   verify-api-extract:
     docker: *ref_4
     working_directory: ~/repo
diff --git a/.codebuild/cleanup_workflow.yml b/.codebuild/cleanup_workflow.yml
@@ -9,4 +9,4 @@ phases:
   build:
     commands:
       - yarn production-build
-      - cd packages/amplify-e2e-tests && yarn clean-cb-e2e-resources
+      - cd packages/amplify-codegen-e2e-tests && yarn clean-cb-e2e-resources
diff --git a/.codebuild/e2e_workflow.yml b/.codebuild/e2e_workflow.yml
@@ -1,8 +1,8 @@
+# auto generated file. DO NOT EDIT manually
 version: 0.2
 env:
   shell: bash
   compute-type: BUILD_GENERAL1_MEDIUM
-
 batch:
   fast-fail: false
   build-graph:
@@ -28,3 +28,85 @@ batch:
         compute-type: BUILD_GENERAL1_MEDIUM
       depend-on:
         - build_linux
+    - identifier: >-
+        add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/add-codegen-ios.test.ts|src/__tests__/configure-codegen-android.test.ts|src/__tests__/configure-codegen-js.test.ts|src/__tests__/graphql-codegen-android.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        graphql_codegen_js_remove_codegen_android_remove_codegen_ios_add_codegen_android
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/graphql-codegen-js.test.ts|src/__tests__/remove-codegen-android.test.ts|src/__tests__/remove-codegen-ios.test.ts|src/__tests__/add-codegen-android.test.ts
+          CLI_REGION: us-east-2
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        configure_codegen_ios_datastore_modelgen_android_datastore_modelgen_js_feature_flags
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/configure-codegen-ios.test.ts|src/__tests__/datastore-modelgen-android.test.ts|src/__tests__/datastore-modelgen-js.test.ts|src/__tests__/feature-flags.test.ts
+          CLI_REGION: us-west-2
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        graphql_codegen_ios_add_codegen_js_datastore_modelgen_ios_remove_codegen_js
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/graphql-codegen-ios.test.ts|src/__tests__/add-codegen-js.test.ts|src/__tests__/datastore-modelgen-ios.test.ts|src/__tests__/remove-codegen-js.test.ts
+          CLI_REGION: eu-west-2
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        datastore_modelgen_flutter_env_codegen_model_introspection_codegen_pull_codegen
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/datastore-modelgen-flutter.test.ts|src/__tests__/env-codegen.test.ts|src/__tests__/model-introspection-codegen.test.ts|src/__tests__/pull-codegen.test.ts
+          CLI_REGION: eu-central-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        push_codegen_ios_push_codegen_android_graphql_documents_generator_push_codegen_js
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/push-codegen-ios.test.ts|src/__tests__/push-codegen-android.test.ts|src/__tests__/graphql-documents-generator.test.ts|src/__tests__/push-codegen-js.test.ts
+          CLI_REGION: ap-northeast-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: build_app_ts
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: src/__tests__/build-app-ts.test.ts
+          CLI_REGION: ap-southeast-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: cleanup_e2e_resources
+      buildspec: .codebuild/cleanup_e2e_resources.yml
+      env:
+        compute-type: BUILD_GENERAL1_SMALL
+      depend-on:
+        - >-
+          add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
diff --git a/.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts b/.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts
@@ -9,10 +9,10 @@
  *
     artifacts:
       files:
-        - $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports/*
+        - $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports/*
  *
  * From the above job, 'path' includes the following:
- *  $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports
+ *  $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports
  *
  * Those paths must be included in this list.
  *
@@ -27,6 +27,6 @@
  * will automatically normalize these paths for Windows if it detects it.
  */
 export const ARTIFACT_STORAGE_PATH_ALLOW_LIST_CODEBUILD = [
-  '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/',
-  '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports'
+  '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/',
+  '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports'
 ];
diff --git a/.codebuild/scripts/local_publish_helpers.sh b/.codebuild/scripts/local_publish_helpers.sh
@@ -44,7 +44,9 @@ function unsetSudoNpmRegistryUrl {
 
 function changeNpmGlobalPath {
     mkdir -p ~/.npm-global
-    npm config set prefix '~/.npm-global'
+    if [ -z $SKIP_SET_NPM_PREFIX ]; then
+        npm config set prefix '~/.npm-global'
+    fi
     export PATH=~/.npm-global/bin:$PATH
 }
 
diff --git a/.gitignore b/.gitignore
@@ -26,4 +26,5 @@ test.out.log
 *.tsbuildinfo
 package-lock.json
 .idea
-scripts/.env
+scripts/.env
+scripts/cci/job.data.json
diff --git a/package.json b/package.json
@@ -39,7 +39,9 @@
     "view-test-artifact": "./scripts/view-test-artifacts.sh",
     "cleanup-stale-resources": "source ./scripts/cloud-utils.sh && cleanupStaleResources",
     "cloud-e2e-cb": "source scripts/cloud-utils.sh && cloudE2E",
-    "cloud-e2e-cb-beta": "source scripts/cloud-utils.sh && cloudE2EBeta"
+    "cloud-e2e-cb-beta": "source scripts/cloud-utils.sh && cloudE2EBeta",
+    "split-codebuild-e2e-tests": "yarn ts-node ./scripts/split-e2e-tests-codebuild.ts && git add .codebuild/e2e_workflow.yml",
+    "update-test-timing-data": "ts-node ./scripts/cci/get-test-timing-from-cci-job-metrics.ts"
   },
   "bugs": {
     "url": "https://github.com/aws-amplify/amplify-codegen/issues"
@@ -55,7 +57,7 @@
     "hooks": {
       "commit-msg": "commitlint -E HUSKY_GIT_PARAMS",
       "pre-push": "npm run lint && npm run test-changed",
-      "pre-commit": "yarn split-e2e-tests && pretty-quick --staged"
+      "pre-commit": "yarn split-codebuild-e2e-tests"
     }
   },
   "author": "Amazon Web Services",
diff --git a/scripts/cci/api.ts b/scripts/cci/api.ts
@@ -0,0 +1,117 @@
+import axios from 'axios';
+
+type ReportingWindow = 'last-7-days' | 'last-90-days' | 'last-24-hours' | 'last-30-days' | 'last-60-days';
+export type CircleCIClientDefaults = {
+  defaultBranch: string;
+  defaultWorkflow: string;
+  vcs: string;
+  projectSlug: string;
+  projectName: string;
+};
+export class CircleCIAPIClient {
+  private headers;
+  private options: CircleCIClientDefaults;
+  private slug: string;
+  constructor(token: string, options: CircleCIClientDefaults) {
+    this.headers = {
+      'Circle-Token': token,
+    };
+    this.options = options;
+    this.slug = `${options.vcs}/${options.projectSlug}/${options.projectName}`;
+  }
+
+  /**
+   * Returns a sequence of jobs for a workflow.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/listWorkflowJobs
+   * @returns
+   */
+  getWorkflowJobs = async (workflowId: string = this.options.defaultWorkflow) => {
+    const result = await axios.get(`https://circleci.com/api/v2/workflow/${workflowId}/job`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Returns a job's details.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getJobDetails
+   * @param jobId
+   * @returns
+   */
+  getJobDetails = async (jobId: string) => {
+    const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/job/${jobId}`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Returns a single job's artifacts.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts
+   * @param jobId
+   * @returns
+   */
+  getJobArtifacts = async (jobId: string) => {
+    const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/${jobId}/artifacts`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Get test metadata for a single job
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getTests
+   * @param jobId
+   * @returns
+   */
+  getJobTests = async (jobId: string) => {
+    const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/${jobId}/tests`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Get summary metrics for a project workflow's jobs.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getProjectWorkflowJobMetrics
+   *
+   * @param workflowName
+   * @param branch
+   * @param reportingWindow
+   * @returns
+   */
+  getAllJobMetrics = async (
+    workflowName: string = this.options.defaultWorkflow,
+    branch: string = this.options.defaultBranch,
+    reportingWindow: ReportingWindow = 'last-30-days',
+  ) => {
+    const result = await axios.get(`https://circleci.com/api/v2/insights/${this.slug}/workflows/${workflowName}/jobs`, {
+      headers: this.headers,
+      params: {
+        branch: branch,
+        'reporting-window': reportingWindow,
+      },
+    });
+    return result.data;
+  };
+
+  /**
+   * Get test metrics for a project's workflows.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getProjectWorkflowTestMetrics
+   * @param workflowName
+   * @param branch
+   * @param reportingWindow
+   * @returns
+   */
+  getAllTestMetrics = async (workflowName: string = this.options.defaultWorkflow, branch: string = this.options.defaultBranch) => {
+    const result = await axios.get(`https://circleci.com/api/v2/insights/${this.slug}/workflows/${workflowName}/test-metrics`, {
+      headers: this.headers,
+      params: {
+        branch: branch,
+      },
+    });
+    return result.data;
+  };
+}
diff --git a/scripts/cci/get-test-timing-from-cci-job-metrics.ts b/scripts/cci/get-test-timing-from-cci-job-metrics.ts
@@ -0,0 +1,37 @@
+import { join } from 'path';
+import { REPO_ROOT, getCCIClient, getTestFiles, getTestNameFromPath, getTimingsFromJobsData, saveJobMetrics, saveTestTimings } from './utils';
+
+async function main(): Promise<void> {
+  const client = getCCIClient();
+  console.log('Fetching job metrics...');
+  const data = await client.getAllJobMetrics();
+  saveJobMetrics(data);
+
+  const testSuites = getTestFiles(join(REPO_ROOT, 'packages', 'amplify-codegen-e2e-tests'));
+
+  const jobTimings = getTimingsFromJobsData();
+  const testRuntimes = testSuites.map(t => {
+    const oldName = getTestNameFromPath(t);
+    if (jobTimings.has(oldName)) {
+      return {
+        test: t,
+        medianRuntime: jobTimings.get(oldName) as number,
+      };
+    } else {
+      console.log('Could not find timing for:', t);
+      return {
+        test: t,
+        medianRuntime: 10, // default for unknown
+      };
+    }
+  });
+  testRuntimes.sort((a, b) => {
+    return a.medianRuntime - b.medianRuntime;
+  });
+  saveTestTimings({
+    lastUpdated: new Date().toISOString(),
+    totalTestFiles: testRuntimes.length,
+    timingData: testRuntimes,
+  });
+}
+main();
diff --git a/scripts/cci/test-timings.data.json b/scripts/cci/test-timings.data.json
diff --git a/scripts/cci/utils.ts b/scripts/cci/utils.ts
diff --git a/scripts/split-e2e-tests-codebuild.ts b/scripts/split-e2e-tests-codebuild.ts
diff --git a/shared-scripts.sh b/shared-scripts.sh

Original file line number	Diff line number	Diff line change
`@@ -9,10 +9,10 @@`
`9`	`9`	`*`
`10`	`10`	`artifacts:`
`11`	`11`	`files:`
`12`		`- - $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports/*`
	`12`	`+ - $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports/*`
`13`	`13`	`*`
`14`	`14`	`* From the above job, 'path' includes the following:`
`15`		`- * $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports`
	`15`	`+ * $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports`
`16`	`16`	`*`
`17`	`17`	`* Those paths must be included in this list.`
`18`	`18`	`*`
`@@ -27,6 +27,6 @@`
`27`	`27`	`* will automatically normalize these paths for Windows if it detects it.`
`28`	`28`	`*/`
`29`	`29`	`export const ARTIFACT_STORAGE_PATH_ALLOW_LIST_CODEBUILD = [`
`30`		`- '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/',`
`31`		`- '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports'`
	`30`	`+ '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/',`
	`31`	`+ '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports'`
`32`	`32`	`];`
Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,9 @@ function unsetSudoNpmRegistryUrl {`
`44`	`44`
`45`	`45`	`function changeNpmGlobalPath {`
`46`	`46`	`mkdir -p ~/.npm-global`
`47`		`- npm config set prefix '~/.npm-global'`
	`47`	`+ if [ -z $SKIP_SET_NPM_PREFIX ]; then`
	`48`	`+ npm config set prefix '~/.npm-global'`
	`49`	`+ fi`
`48`	`50`	`export PATH=~/.npm-global/bin:$PATH`
`49`	`51`	`}`
`50`	`52`