do not include flaky tests in matrix runs

Author: sdebruyn

.github/workflows/integration-tests-azure.yml

@@ -70,4 +70,36 @@ jobs:
           DBT_TEST_AAD_PRINCIPAL_1: ${{ secrets.DBT_TEST_AAD_PRINCIPAL_1 }}
           DBT_TEST_AAD_PRINCIPAL_2: ${{ secrets.DBT_TEST_AAD_PRINCIPAL_2 }}
           SQLSERVER_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server'
-        run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}"
+        run: pytest --ignore=tests/functional/adapter/test_provision_users.py -ra -v tests/functional --profile "${{ matrix.profile }}"
+
+  flaky-tests-azure:
+    name: Integration tests on Azure
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/${{ github.repository }}:CI-3.11-msodbc18
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: pip install -r dev_requirements.txt
+
+      - name: Wake up server
+        env:
+          DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }}
+          DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }}
+          DBT_AZURESQL_UID: ${{ secrets.DBT_AZURESQL_UID }}
+          DBT_AZURESQL_PWD: ${{ secrets.DBT_AZURESQL_PWD }}
+          MSODBC_VERSION: ${{ matrix.msodbc_version }}
+        run: python devops/scripts/wakeup_azure.py
+
+      - name: Run auto provisioning tests
+        env:
+          DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }}
+          DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }}
+          AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }}
+          AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }}
+          AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }}
+          DBT_TEST_AAD_PRINCIPAL_1: ${{ secrets.DBT_TEST_AAD_PRINCIPAL_1 }}
+          DBT_TEST_AAD_PRINCIPAL_2: ${{ secrets.DBT_TEST_AAD_PRINCIPAL_2 }}
+          SQLSERVER_TEST_DRIVER: 'ODBC Driver 18 for SQL Server'
+        run: pytest -ra -v tests/functional/adapter/test_provision_users.py --profile "ci_azure_environment"

tests/functional/adapter/test_provision_users.py

@@ -60,9 +60,7 @@ def test_auto_provision(self, project):
         run_dbt(["run"])
 
 
-@pytest.mark.only_with_profile(
-    "ci_azure_cli", "ci_azure_auto", "ci_azure_environment", "user_azure"
-)
+@pytest.mark.only_with_profile("ci_azure_environment", "user_azure")
 @pytest.mark.flaky(max_runs=5, min_passes=1)
 class TestProvisionSingleUserAzureSQL(BaseTestProvisionAzureSQL):
     @pytest.fixture(scope="class")
@@ -73,9 +71,7 @@ def models(self):
         }
 
 
-@pytest.mark.only_with_profile(
-    "ci_azure_cli", "ci_azure_auto", "ci_azure_environment", "user_azure"
-)
+@pytest.mark.only_with_profile("ci_azure_environment", "user_azure")
 @pytest.mark.flaky(max_runs=5, min_passes=1)
 class TestProvisionMultipleUsersAzureSQL(BaseTestProvisionAzureSQL):
     @pytest.fixture(scope="class")