master merge for 1.12.0 release

.github/workflows/lint.yml

@@ -2,25 +2,13 @@
 name: lint | code & tests
 
 on:
-  pull_request:
-    branches:
-      - master
-      - devel
+  workflow_call:
   workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 jobs:
-  get_docs_changes:
-    name: docs changes
-    uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
     name: lint
-    needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: true
       matrix:
@@ -60,7 +48,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --all-extras --with airflow,providers,pipeline,sentry-sdk,dbt
+        run: poetry install --all-extras --with airflow,providers,pipeline,sentry-sdk,dbt,marimo,streamlit
 
       - name: Run make lint
         run: |

.github/workflows/main.yml

@@ -0,0 +1,105 @@
+name: dlt
+
+on:
+  pull_request:
+    branches: [ master, devel ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  # checks wether anything outside docs has changed, if not, most of the tests can be skipped
+  # lint will check this and if lint is not run, no other tests will be run
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  # if the PR is from a fork, we need to authorize the secrets access and remote destinations run with a lable
+  authorize_run_from_fork:
+    name: check if fork and if so wether secrets are available
+    # run when label is assigned OR when we are not a fork
+    if: ${{ github.event.label.name == 'ci from fork' || (github.event.pull_request.head.repo.full_name == github.repository && (github.event.action == 'opened' || github.event.action == 'synchronize'))}}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
+
+  # testing and linting of docs snippets is always run to catch problems in the docs
+  # NOTE: we could splint linting and testing of docs so linter can also always run for fork PRs
+  test_docs_snippets:
+    name: test snippets in docs
+    uses: ./.github/workflows/test_docs_snippets.yml
+    secrets: inherit
+
+  lint:
+    name: lint on all python versions
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    uses: ./.github/workflows/lint.yml
+
+  test_common:
+    name: run common tests on all python versions and OSes
+    needs: lint
+    uses: ./.github/workflows/test_common.yml
+
+  #
+  # Destination and Sources local tests, do not provide secrets
+  # Other tests that do not require remote connections
+  #
+  test_destinations_local:
+    name: run local destination tests without secrets
+    needs: test_common
+    uses: ./.github/workflows/test_destinations_local.yml
+
+  test_sources_local:
+    name: run local source tests without secrets
+    needs: test_common
+    uses: ./.github/workflows/test_sources_local.yml
+
+  test_plus:
+    name: test dlt+ connection
+    needs: lint
+    uses: ./.github/workflows/test_plus.yml
+
+  test_tools_airflow:
+    name: test airflow helpers
+    needs: lint
+    uses: ./.github/workflows/test_tools_airflow.yml
+
+  test_tools_build_images:
+    name: test build images
+    needs: lint
+    uses: ./.github/workflows/test_tools_build_images.yml
+
+  #
+  # Remote destination tests and docs snippets, needs secrets, 
+  # so we depend on authorize and forward secrets
+  #
+
+
+  test_destinations_remote:
+    name: test remote destinations with secrets
+    needs: [authorize_run_from_fork, test_common]
+    uses: ./.github/workflows/test_destinations_remote.yml
+    secrets: inherit
+    with:
+      run_full_test_suite: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
+
+  #
+  # Other tools and tests that require secrets
+  #
+  test_tools_dbt_runner:
+    name: test dbt runner
+    needs: [test_common, authorize_run_from_fork]
+    uses: ./.github/workflows/test_tools_dbt_runner.yml
+    secrets: inherit
+
+  # dbt cloud tests currently are disabled, TODO: explain why
+  # test_tools_dbt_cloud:
+  #   needs: [test_common, authorize_run_from_fork]
+  #   uses: ./.github/workflows/test_tools_dbt_cloud.yml
+  #   secrets: inherit
+
+
+

.github/workflows/test_common.yml

@@ -1,19 +1,11 @@
 name: common | common
 
 on:
-  pull_request:
-    branches:
-      - master
-      - devel
-  workflow_dispatch:
+  workflow_call:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
 
 env:
   RUNTIME__LOG_LEVEL: ERROR
-  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
   # we need the secrets only for the rest_api_pipeline tests which are in tests/sources
   # so we inject them only at the end
@@ -22,35 +14,48 @@ env:
   SOURCES__GITHUB_API_PIPELINE__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  get_docs_changes:
-    name: docs changes
-    uses: ./.github/workflows/get_docs_changes.yml
-
   run_common:
     name: test
-    needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.11.x"]
-        # Test all python versions on ubuntu only
         include:
-          - python-version: "3.9.x"
-            os: "ubuntu-latest"
-          - python-version: "3.10.x"
-            os: "ubuntu-latest"
-          - python-version: "3.12.x"
-            os: "ubuntu-latest"
-          - python-version: "3.13.x"
-            os: "ubuntu-latest"
-          - python-version: "3.13.x"
-            os: "windows-latest"
+
+          # macos tests
+          - os: macos-latest
+            python-version: "3.11.x"
+            shell: bash
+
+          # linux tests
+          - os: ubuntu-latest
+            python-version: "3.9.x"
+            shell: bash
+          - os: ubuntu-latest
+            python-version: "3.10.x"
+            shell: bash
+          - os: ubuntu-latest
+            python-version: "3.11.x"
+            shell: bash
+          - os: ubuntu-latest
+            python-version: "3.12.x"
+            shell: bash
+          - os: ubuntu-latest
+            python-version: "3.13.x"
+            shell: bash
+
+          # windows tests
+          - os: windows-latest
+            python-version: "3.11.x"
+            shell: cmd
+            pytest_args: '-m "not forked"'
+          - os: windows-latest
+            python-version: "3.13.x"
+            shell: cmd
+            pytest_args: '-m "not forked"'
 
     defaults:
       run:
-        shell: bash
+        shell: ${{ matrix.shell }}
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -72,7 +77,6 @@ jobs:
           copy tzdata %USERPROFILE%\Downloads\tzdata
           curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml
         if: runner.os == 'Windows'
-        shell: cmd
 
       - name: Install Poetry
         # https://github.com/snok/install-poetry#running-on-windows
@@ -83,93 +87,67 @@ jobs:
           installer-parallel: true
           version: 1.8.5
 
-      # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
-      # - name: Load cached venv
-      #   id: cached-poetry-dependencies
-      #   uses: actions/cache@v3
-      #   with:
-      #     # path: ${{ steps.pip-cache.outputs.dir }}
-      #     path: .venv
-      #     key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
       - name: Install dependencies
         run: poetry install --no-interaction --with sentry-sdk
-
-      - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py
-        if: runner.os != 'Windows'
-        name: Run common tests with minimum dependencies Linux/MAC
-      - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked"
-        if: runner.os == 'Windows'
-        name: Run common tests with minimum dependencies Windows
-        shell: cmd
+
+      - name: Run common tests with minimum dependencies
+        run: |
+          poetry run pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py ${{ matrix.pytest_args }}
+
 
       - name: Install duckdb dependencies
         run: poetry install --no-interaction -E duckdb --with sentry-sdk
 
-      - run: |
-          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py
-        if: runner.os != 'Windows'
-        name: Run pipeline smoke tests with minimum deps Linux/MAC
-      - run: |
-          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py -m "not forked"
-        if: runner.os == 'Windows'
-        name: Run smoke tests with minimum deps Windows
-        shell: cmd
+      - name: Run pipeline smoke tests with minimum deps
+        run: |
+          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py ${{ matrix.pytest_args }}
 
       - name: Install pyarrow
         run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk
 
-      - run: |
-          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
-        if: runner.os != 'Windows'
-        name: Run pipeline tests with pyarrow but no pandas installed
-      - run: |
-          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow  -m "not forked"
-        if: runner.os == 'Windows'
-        name: Run pipeline tests with pyarrow but no pandas installed Windows
-        shell: cmd
+      - name: Run pipeline tests with pyarrow but no pandas installed
+        run: |
+          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow ${{ matrix.pytest_args }}
 
       - name: Install pipeline and sources dependencies
         run: poetry install --no-interaction -E duckdb -E cli -E parquet -E deltalake -E sql_database --with sentry-sdk,pipeline,sources
 
-      - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources
-        if: runner.os != 'Windows'
-        name: Run extract and pipeline tests Linux/MAC
-      - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources -m "not forked"
-        if: runner.os == 'Windows'
-        name: Run extract tests Windows
-        shell: cmd
+      - name: Run extract and pipeline tests
+        run: |
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources tests/transformations ${{ matrix.pytest_args }}
 
       # here we upgrade sql alchemy to 2 an run the sql_database tests again
       - name: Upgrade sql alchemy
         run: poetry run pip install sqlalchemy==2.0.32
 
-      - run: |
-          poetry run pytest tests/sources/sql_database
-        if: runner.os != 'Windows'
-        name: Run extract and pipeline tests Linux/MAC
-      - run: |
+      - name: Run extract and pipeline tests 
+        run: |
           poetry run pytest tests/sources/sql_database
-        if: runner.os == 'Windows'
-        name: Run extract tests Windows
-        shell: cmd
-
-      # - name: Install Pydantic 1.0
-      #   run: pip install "pydantic<2"
-
-      # - run: |
-      #     poetry run pytest tests/libs
-      #   if: runner.os != 'Windows'
-      #   name: Run extract and pipeline tests Linux/MAC
-      # - run: |
-      #     poetry run pytest tests/libs
-      #   if: runner.os == 'Windows'
-      #   name: Run extract tests Windows
-      #   shell: cmd
+
+
+
+      # test marimo app, does not work with python 3.13
+      - name: Install dlt with duckdb and studio
+        run: poetry install --no-interaction -E duckdb --with sentry-sdk,pipeline,sources,ibis,marimo
+        if: matrix.python-version != '3.13.x'
+
+      - name: Install playwright
+        run: poetry run playwright install
+        if: matrix.python-version != '3.13.x'
+
+      # Run marimo studio unit tests
+      - name: Run marimo studio unit tests
+        run: |
+          poetry run pytest tests/helpers/studio
+        if: matrix.python-version != '3.13.x'
+
+      # Run marimo e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
+      - name: Run marimo e2e
+        run: |
+          poetry run marimo run --headless dlt/helpers/studio/app.py -- -- --pipelines_dir _storage/.dlt/pipelines/ --with_test_identifiers true & poetry run pytest --browser chromium tests/e2e
+        if: matrix.python-version != '3.13.x' && matrix.python-version != '3.9.x' && matrix.os != 'windows-latest'
+
+
 
   matrix_job_required_check:
     name: common | common tests