Update github workflow setup

.github/workflows/lint.yml

@@ -2,25 +2,13 @@
 name: lint | code & tests
 
 on:
-  pull_request:
-    branches:
-      - master
-      - devel
+  workflow_call:
   workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 jobs:
-  get_docs_changes:
-    name: docs changes
-    uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
     name: lint
-    needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: true
       matrix:

.github/workflows/main.yml

@@ -0,0 +1,102 @@
+name: dlt
+
+on:
+  pull_request:
+    branches: [ master, devel ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  # checks wether anything outside docs has changed, if not, most of the tests can be skipped
+  # lint will check this and if lint is not run, no other tests will be run
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  # if the PR is from a fork, we need to authorize the secrets access and remote destinations run with a lable
+  authorize_run_from_fork:
+    name: check if fork and if so wether secrets are available
+    # run when label is assigned OR when we are not a fork
+    if: ${{ github.event.label.name == 'ci from fork' || (github.event.pull_request.head.repo.full_name == github.repository && (github.event.action == 'opened' || github.event.action == 'synchronize'))}}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
+
+  lint:
+    name: lint on all python versions
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    uses: ./.github/workflows/lint.yml
+
+  test_common:
+    name: run common tests on all python versions and OSes
+    needs: lint
+    uses: ./.github/workflows/test_common.yml
+
+  #
+  # Destination and Sources local tests, do not provide secrets
+  # Other tests that do not require remote connections
+  #
+  test_destinations_local:
+    name: run local destination tests without secrets
+    needs: test_common
+    uses: ./.github/workflows/test_destinations_local.yml
+
+  test_sources_local:
+    name: run local source tests without secrets
+    needs: test_common
+    uses: ./.github/workflows/test_sources_local.yml
+
+  test_plus:
+    name: test dlt+ connection
+    needs: lint
+    uses: ./.github/workflows/test_plus.yml
+
+  test_tools_airflow:
+    name: test airflow helpers
+    needs: lint
+    uses: ./.github/workflows/test_tools_airflow.yml
+
+  test_tools_build_images:
+    name: test build images
+    needs: lint
+    uses: ./.github/workflows/test_tools_build_images.yml
+
+  #
+  # Remote destination tests and docs snippets, needs secrets, 
+  # so we depend on authorize and forward secrets
+  #
+  test_docs_snippets:
+    name: test snippets in docs
+    needs: [test_common, authorize_run_from_fork]
+    uses: ./.github/workflows/test_docs_snippets.yml
+    secrets: inherit
+
+  test_destinations_remote:
+    name: test remote destinations with secrets
+    needs: [authorize_run_from_fork, test_common]
+    uses: ./.github/workflows/test_destinations_remote.yml
+    secrets: inherit
+    with:
+      run_full_test_suite: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
+
+  #
+  # Other tools and tests that require secrets
+  #
+  test_tools_dbt_runner:
+    name: test dbt runner
+    needs: [test_common, authorize_run_from_fork]
+    uses: ./.github/workflows/test_tools_dbt_runner.yml
+    secrets: inherit
+
+  # dbt cloud tests currently are disabled, TODO: explain why
+  # test_tools_dbt_cloud:
+  #   needs: [test_common, authorize_run_from_fork]
+  #   uses: ./.github/workflows/test_tools_dbt_cloud.yml
+  #   secrets: inherit
+
+
+

.github/workflows/test_common.yml

@@ -1,19 +1,11 @@
 name: common | common
 
 on:
-  pull_request:
-    branches:
-      - master
-      - devel
-  workflow_dispatch:
+  workflow_call:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
 
 env:
   RUNTIME__LOG_LEVEL: ERROR
-  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
   # we need the secrets only for the rest_api_pipeline tests which are in tests/sources
   # so we inject them only at the end
@@ -22,14 +14,8 @@ env:
   SOURCES__GITHUB_API_PIPELINE__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  get_docs_changes:
-    name: docs changes
-    uses: ./.github/workflows/get_docs_changes.yml
-
   run_common:
     name: test
-    needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -39,29 +25,23 @@ jobs:
           - os: macos-latest
             python-version: "3.11.x"
             shell: bash
-            pytest_args: ""
 
           # linux tests
           - os: ubuntu-latest
             python-version: "3.9.x"
             shell: bash
-            pytest_args: ""
           - os: ubuntu-latest
             python-version: "3.10.x"
             shell: bash
-            pytest_args: ""
           - os: ubuntu-latest
             python-version: "3.11.x"
             shell: bash
-            pytest_args: ""
           - os: ubuntu-latest
             python-version: "3.12.x"
             shell: bash
-            pytest_args: ""
           - os: ubuntu-latest
             python-version: "3.13.x"
             shell: bash
-            pytest_args: "" 
 
           # windows tests
           - os: windows-latest
@@ -107,20 +87,9 @@ jobs:
           installer-parallel: true
           version: 1.8.5
 
-
-      # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
-      # - name: Load cached venv
-      #   id: cached-poetry-dependencies
-      #   uses: actions/cache@v3
-      #   with:
-      #     # path: ${{ steps.pip-cache.outputs.dir }}
-      #     path: .venv
-      #     key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-
       - name: Install dependencies
         run: poetry install --no-interaction --with sentry-sdk
-
+    
       - name: Run common tests with minimum dependencies
         run: |
           poetry run pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py ${{ matrix.pytest_args }}
@@ -154,14 +123,7 @@ jobs:
       - name: Run extract and pipeline tests 
         run: |
           poetry run pytest tests/sources/sql_database
-
-
-      # - name: Install Pydantic 1.0
-      #   run: pip install "pydantic<2"
 
-      # - name: Run extract and pipeline tests
-      #   run: |
-      #     poetry run pytest tests/libs
 
 
       # test marimo app, does not work with python 3.13