Parallelizing CI tests

.github/actions/test-provider-tfe/action.yml

@@ -87,7 +87,6 @@ runs:
         index: ${{ inputs.matrix_index }}
         total: ${{ inputs.matrix_total }}
         junit-summary: ./ci-summary-provider.xml
-        # When tests are split and run concurrently, lists_tests arg in ci.yml will skip the TestAccTFESAMLSettings_omnibus test suite
         list: ${{ inputs.list_tests }}
 
     - name: Run Tests
@@ -121,6 +120,6 @@ runs:
     - name: Upload test artifacts
       uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
       with:
-        name: junit-test-summary-${{ matrix.index }}
+        name: junit-${{ matrix.test_name }}-summary-${{ matrix.index }}
         path: summary.xml
         retention-days: 1

.github/workflows/ci.yml

@@ -17,15 +17,90 @@ jobs:
       - uses: ./.github/actions/lint-provider-tfe
 
   tests:
-    name: run
+    name: tests
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
-        # If you adjust these parameters, also adjust the jrm input files on the "Merge reports" step below
-        total: [ 1 ]
-        index: [ 0 ]
+        total: [5]
+        index: [0, 1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Fetch Outputs
+        id: tflocal
+        uses: hashicorp-forge/terraform-cloud-action/outputs@5583d5f554d268ac91b3c37fd0a5e9da2c78c017 # v1.1.0
+        with:
+          token: "${{ secrets.TF_WORKFLOW_TFLOCAL_CLOUD_TFC_TOKEN }}"
+          organization: hashicorp-v2
+          workspace: tflocal-terraform-provider-tfe
+
+
+      - uses: ./.github/actions/test-provider-tfe
+        with:
+          matrix_index: ${{ matrix.index }}
+          matrix_total: ${{ matrix.total }}
+          hostname: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).ngrok_domain }}
+          token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_token }}
+          testing-github-token: ${{ secrets.TESTING_GITHUB_TOKEN }}
+          admin_configuration_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.configuration }}
+          admin_provision_licenses_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.provision-licenses }}
+          admin_security_maintenance_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.security-maintenance }}
+          admin_site_admin_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.site-admin }}
+          admin_subscription_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.subscription }}
+          admin_support_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.support }}
+          admin_version_maintenance_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.version-maintenance }}
+          list_tests: "[^(TestAccTFESAMLSettings_omnibus|TestAcc.*RunDependent|TestAcc.*VCSDependent)]"
+
+  run-dependent-tests:
+    name: run-dependent-tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        total: [1]
+        index: [0]
+    steps:
+      - name: Fetch Outputs
+        id: tflocal
+        uses: hashicorp-forge/terraform-cloud-action/outputs@5583d5f554d268ac91b3c37fd0a5e9da2c78c017 # v1.1.0
+        with:
+          token: "${{ secrets.TF_WORKFLOW_TFLOCAL_CLOUD_TFC_TOKEN }}"
+          organization: hashicorp-v2
+          workspace: tflocal-terraform-provider-tfe
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: ./.github/actions/test-provider-tfe
+        with:
+          matrix_index: ${{ matrix.index }}
+          matrix_total: ${{ matrix.total }}
+          hostname: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).ngrok_domain }}
+          token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_token }}
+          testing-github-token: ${{ secrets.TESTING_GITHUB_TOKEN }}
+          admin_configuration_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.configuration }}
+          admin_provision_licenses_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.provision-licenses }}
+          admin_security_maintenance_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.security-maintenance }}
+          admin_site_admin_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.site-admin }}
+          admin_subscription_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.subscription }}
+          admin_support_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.support }}
+          admin_version_maintenance_token: ${{ fromJSON(steps.tflocal.outputs.workspace-outputs-json).tfe_admin_token_by_role.version-maintenance }}
+          # Run terminal cmd 'go help testflag' to learn more about -list flag
+          # action.yml uses https://github.com/hashicorp-forge/go-test-split-action/blob/main/action.yml to split acceptance tests
+          # which runs against all tests using the list arg
+          # lists_tests regex is used to skip the TestAccTFESAMLSettings_omnibus test suite for CI tests only
+          list_tests: "TestAcc.*_RunDependent"
+  vcs-dependent-tests:
+    name: vcs-dependent-tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        total: [1]
+        index: [0]
     steps:
       - name: Fetch Outputs
         id: tflocal
@@ -55,11 +130,11 @@ jobs:
           # action.yml uses https://github.com/hashicorp-forge/go-test-split-action/blob/main/action.yml to split acceptance tests
           # which runs against all tests using the list arg
           # lists_tests regex is used to skip the TestAccTFESAMLSettings_omnibus test suite for CI tests only
-          list_tests: "[^(TestAccTFESAMLSettings_omnibus)]"
+          list_tests: "TestAcc.*_VCSDependent"
 
   tests-combine-summaries:
     name: Combine Test Reports
-    needs: [ tests ]
+    needs: [ tests, run-dependent-tests, vcs-dependent-tests ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
@@ -73,7 +148,8 @@ jobs:
         run: npm install -g junit-report-merger
 
       - name: Merge reports
-        run: jrm ./ci-summary-provider.xml "junit-test-summary-0/*.xml" "junit-test-summary-1/*.xml" "junit-test-summary-2/*.xml" "junit-test-summary-3/*.xml" "junit-test-summary-4/*.xml"
+        run: |
+          jrm ./ci-summary-provider.xml junit-tests-summary-*/*.xml
 
       - name: Upload test artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -82,14 +158,14 @@ jobs:
           path: ./ci-summary-provider.xml
 
   tests-summarize:
-    name: Summarize Tests
-    needs: [ tests ]
+    name: tests-summarize
+    needs: [ tests, run-dependent-tests, vcs-dependent-tests ]
     runs-on: ubuntu-latest
     if: ${{ always() }}
     steps:
       - name: Check tests Status
         run: |
-          if [ "${{ needs.tests.result }}" = "success" ]; then
+          if [ "${{ needs.tests.result }}" = "success" ] && [ "${{ needs.run-dependent-tests.result }}" = "success" ]; then
             exit 0
           fi
           exit 1

internal/provider/data_source_policy_set_test.go

@@ -163,7 +163,7 @@ func TestAccTFEPolicySetDataSourceOPA_basic(t *testing.T) {
 	)
 }
 
-func TestAccTFEPolicySetDataSource_vcs(t *testing.T) {
+func TestAccTFEPolicySetDataSource_vcs_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatal(err)

internal/provider/data_source_registry_module_test.go

@@ -92,7 +92,7 @@ func TestAccTFERegistryModuleDataSource_basicNoCodePublic(t *testing.T) {
 	})
 }
 
-func TestAccTFERegistryModuleDataSource_basicNoCodePrivate(t *testing.T) {
+func TestAccTFERegistryModuleDataSource_basicNoCodePrivate_VCSDependent(t *testing.T) {
 	skipIfEnterprise(t)
 
 	dsName := "data.tfe_registry_module.test"

internal/provider/resource_tfe_no_code_module_test.go

@@ -41,7 +41,7 @@ func TestAccTFENoCodeModule_basic(t *testing.T) {
 	})
 }
 
-func TestAccTFENoCodeModule_with_variable_options(t *testing.T) {
+func TestAccTFENoCodeModule_with_variable_options_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatalf("error getting client %v", err)
@@ -98,7 +98,7 @@ func TestAccTFENoCodeModule_with_variable_options(t *testing.T) {
 	})
 }
 
-func TestAccTFENoCodeModule_with_version_pin(t *testing.T) {
+func TestAccTFENoCodeModule_with_version_pin_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatalf("error getting client %v", err)

internal/provider/resource_tfe_policy_set_test.go

@@ -471,7 +471,7 @@ func TestAccTFEPolicySet_updateToWorkspace(t *testing.T) {
 	})
 }
 
-func TestAccTFEPolicySet_vcs(t *testing.T) {
+func TestAccTFEPolicySet_vcs_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatal(err)
@@ -526,7 +526,7 @@ func TestAccTFEPolicySet_vcs(t *testing.T) {
 	})
 }
 
-func TestAccTFEPolicySet_GithubApp(t *testing.T) {
+func TestAccTFEPolicySet_GithubApp_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatal(err)
@@ -581,7 +581,7 @@ func TestAccTFEPolicySet_GithubApp(t *testing.T) {
 	})
 }
 
-func TestAccTFEPolicySet_updateVCSBranch(t *testing.T) {
+func TestAccTFEPolicySet_updateVCSBranch_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatal(err)
@@ -767,7 +767,7 @@ func TestAccTFEPolicySet_versionedSlugUpdate(t *testing.T) {
 	})
 }
 
-func TestAccTFEPolicySet_versionedNoConflicts(t *testing.T) {
+func TestAccTFEPolicySet_versionedNoConflicts_VCSDependent(t *testing.T) {
 	tfeClient, err := getClientUsingEnv()
 	if err != nil {
 		t.Fatal(err)