From f37a869c06b826f44e047e676079a081b22ef2c3 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Wed, 12 Mar 2025 11:36:17 +0000 Subject: [PATCH 1/5] Run github test pipeline in parallel --- .github/scripts/run_tests.sh | 25 ------------------ .github/workflows/ci.yml | 49 ++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 35 deletions(-) delete mode 100644 .github/scripts/run_tests.sh diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh deleted file mode 100644 index 9b2282a7..00000000 --- a/.github/scripts/run_tests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -mkdir -p reports - -echo "Starting tests..." - -test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u) - -echo "Test directories found:" -echo "$test_dirs" - -# Run tests in each directory and save reports -for dir in $test_dirs; do - echo "Running tests in directory: $dir" - - # Generate a safe report filename - report_file="reports/$(echo "$dir" | tr '/' '_').txt" - - echo "Saving report to: $report_file" - - pytest "$dir" | tee "$report_file" -done - -echo "All tests executed successfully." \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e1ec8e..fb5d375e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,25 +75,55 @@ jobs: - name: Create .env file for tests run: bash .github/scripts/create_env_file.sh + - name: Discover Test Directories + id: discover-tests + run: | + test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "test_dirs=$test_dirs" >> $GITHUB_OUTPUT + + run-parallel-tests: + name: Run Parallel Tests + runs-on: ubuntu-latest + needs: run-tests + strategy: + matrix: + test_dir: ${{ fromJson(needs.run-tests.outputs.test_dirs) }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set Up Python Environment + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install Python Dependencies + run: | + pip install --upgrade pip + pip install -r dev_requirements.txt + pip install . + - name: Run tests - run: bash .github/scripts/run_tests.sh + run: | + mkdir -p reports + report_file="reports/${{ matrix.test_dir }}.txt" + pytest ${{ matrix.test_dir }} | tee $report_file - - name: Upload tests report as artifact + - name: Upload test report uses: actions/upload-artifact@v4 with: - name: all-tests-reports - path: reports/ + name: ${{ matrix.test_dir }}-report + path: reports/${{ matrix.test_dir }}.txt upload-individual-test-reports: name: Upload Tests Artifacts runs-on: ubuntu-latest - needs: run-tests - + needs: run-parallel-tests steps: - name: Download test reports uses: actions/download-artifact@v4 with: - name: all-tests-reports + name: ${{ matrix.test_dir }}-report path: reports/ - name: Upload individual test reports @@ -105,8 +135,7 @@ jobs: verify-failures: name: Verify Expected Test Failures runs-on: ubuntu-latest - needs: [run-tests, upload-individual-test-reports] - + needs: [run-parallel-tests, upload-individual-test-reports] steps: - name: Check out repository uses: actions/checkout@v3.5.2 @@ -125,7 +154,7 @@ jobs: - name: Extract Actual Failed Tests run: | shopt -s globstar - grep "FAILED" reports/**/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt + grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt - name: Sort Expected Failures run: sort .github/expected_failures.txt > expected_failures_sorted.txt From d256637a00007c7f168a3bef000c25ebe6b5b0a7 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Wed, 12 Mar 2025 16:48:49 +0000 Subject: [PATCH 2/5] Adjusting the CI workflow to run tests in parallel --- .github/workflows/ci.yml | 118 ++++++++---------- .../test_base_mat.py | 0 .../test_ephemeral.py | 0 .../test_incremental.py | 0 .../test_singular_ephemeral.py | 0 .../test_snapshots.py | 0 6 files changed, 49 insertions(+), 69 deletions(-) rename tests/functional/adapter/{basic => materialization}/test_base_mat.py (100%) rename tests/functional/adapter/{basic => materialization}/test_ephemeral.py (100%) rename tests/functional/adapter/{basic => materialization}/test_incremental.py (100%) rename tests/functional/adapter/{basic => materialization}/test_singular_ephemeral.py (100%) rename tests/functional/adapter/{basic => materialization}/test_snapshots.py (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb5d375e..18184000 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,25 @@ on: - main jobs: - run-tests: - name: Setup and Run Tests + discover-tests: + name: Discover Tests runs-on: ubuntu-latest + outputs: + test_dirs: ${{ steps.discover-tests.outputs.test_dirs }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Discover Test Directories + id: discover-tests + run: | + test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "test_dirs=$test_dirs" >> $GITHUB_OUTPUT + + run-parallel-tests: + name: Run Parallel Tests + runs-on: ubuntu-latest + needs: discover-tests env: RETRY_COUNT: 12 # number of retries for health checks SLEEP_INTERVAL: 5 # Sleep duration in seconds between retries @@ -27,21 +43,31 @@ jobs: DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 - + strategy: + matrix: + test_dir: ${{ fromJson(needs.discover-tests.outputs.test_dirs) }} + services: + minio: + image: minio/minio:edge-cicd + options: --network-alias minio + ports: + - 9000:9000 + - 9001:9001 + env: + MINIO_ROOT_USER: ${{ env.MINIO_ROOT_USER }} + MINIO_ROOT_PASSWORD: ${{ env.MINIO_ROOT_PASSWORD }} + dremio: + image: dremio/dremio-oss + options: --network-alias dremio + ports: + - 31010:31010 + - 9047:9047 + env: + DREMIO_JAVA_SERVER_EXTRA_OPTS: -Ddebug.addDefaultUser=true steps: - name: Check out repository uses: actions/checkout@v4 - - name: Create Docker Network - run: | - docker network create ci-network || echo "Network already exists" - - - name: Start MinIO Service - run: bash .github/scripts/start_minio.sh - - - name: Start Dremio Service - run: bash .github/scripts/start_dremio.sh - - name: Install MinIO Client (mc) run: bash .github/scripts/install_minio_client.sh @@ -75,67 +101,27 @@ jobs: - name: Create .env file for tests run: bash .github/scripts/create_env_file.sh - - name: Discover Test Directories - id: discover-tests - run: | - test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "test_dirs=$test_dirs" >> $GITHUB_OUTPUT - - run-parallel-tests: - name: Run Parallel Tests - runs-on: ubuntu-latest - needs: run-tests - strategy: - matrix: - test_dir: ${{ fromJson(needs.run-tests.outputs.test_dirs) }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set Up Python Environment - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Python Dependencies - run: | - pip install --upgrade pip - pip install -r dev_requirements.txt - pip install . + - name: Sanitize test directory name + id: sanitize + run: echo "sanitized_test_dir=$(echo ${{ matrix.test_dir }} | tr '/' '_')" >> $GITHUB_ENV - name: Run tests + continue-on-error: true run: | - mkdir -p reports - report_file="reports/${{ matrix.test_dir }}.txt" + mkdir reports/ + report_file="reports/${{ env.sanitized_test_dir }}.txt" pytest ${{ matrix.test_dir }} | tee $report_file - - name: Upload test report + - name: Upload test report as artifact uses: actions/upload-artifact@v4 with: - name: ${{ matrix.test_dir }}-report - path: reports/${{ matrix.test_dir }}.txt - - upload-individual-test-reports: - name: Upload Tests Artifacts - runs-on: ubuntu-latest - needs: run-parallel-tests - steps: - - name: Download test reports - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.test_dir }}-report + name: ${{ env.sanitized_test_dir }} path: reports/ - - - name: Upload individual test reports - uses: actions/upload-artifact@v4 - with: - name: individual-test-reports - path: reports/*.txt verify-failures: name: Verify Expected Test Failures runs-on: ubuntu-latest - needs: [run-parallel-tests, upload-individual-test-reports] + needs: run-parallel-tests steps: - name: Check out repository uses: actions/checkout@v3.5.2 @@ -143,18 +129,12 @@ jobs: - name: Download All Test Reports uses: actions/download-artifact@v4 with: - name: all-tests-reports path: reports/ - - name: Set Up Python Environment - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Extract Actual Failed Tests run: | shopt -s globstar - grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt + grep "FAILED tests" reports/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt - name: Sort Expected Failures run: sort .github/expected_failures.txt > expected_failures_sorted.txt diff --git a/tests/functional/adapter/basic/test_base_mat.py b/tests/functional/adapter/materialization/test_base_mat.py similarity index 100% rename from tests/functional/adapter/basic/test_base_mat.py rename to tests/functional/adapter/materialization/test_base_mat.py diff --git a/tests/functional/adapter/basic/test_ephemeral.py b/tests/functional/adapter/materialization/test_ephemeral.py similarity index 100% rename from tests/functional/adapter/basic/test_ephemeral.py rename to tests/functional/adapter/materialization/test_ephemeral.py diff --git a/tests/functional/adapter/basic/test_incremental.py b/tests/functional/adapter/materialization/test_incremental.py similarity index 100% rename from tests/functional/adapter/basic/test_incremental.py rename to tests/functional/adapter/materialization/test_incremental.py diff --git a/tests/functional/adapter/basic/test_singular_ephemeral.py b/tests/functional/adapter/materialization/test_singular_ephemeral.py similarity index 100% rename from tests/functional/adapter/basic/test_singular_ephemeral.py rename to tests/functional/adapter/materialization/test_singular_ephemeral.py diff --git a/tests/functional/adapter/basic/test_snapshots.py b/tests/functional/adapter/materialization/test_snapshots.py similarity index 100% rename from tests/functional/adapter/basic/test_snapshots.py rename to tests/functional/adapter/materialization/test_snapshots.py From 2df74243152259b49fc4e74aa43577c0e87a939c Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Wed, 12 Mar 2025 17:07:28 +0000 Subject: [PATCH 3/5] Minor changes --- .github/scripts/start_dremio.sh | 14 -------------- .github/scripts/start_minio.sh | 18 ------------------ .github/workflows/ci.yml | 2 +- 3 files changed, 1 insertion(+), 33 deletions(-) delete mode 100644 .github/scripts/start_dremio.sh delete mode 100644 .github/scripts/start_minio.sh diff --git a/.github/scripts/start_dremio.sh b/.github/scripts/start_dremio.sh deleted file mode 100644 index 972c97c0..00000000 --- a/.github/scripts/start_dremio.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -e - -echo "Starting Dremio service..." - -docker run -d \ - --network ci-network \ - --name dremio \ - -p 31010:31010 \ - -p 9047:9047 \ - -e "DREMIO_JAVA_SERVER_EXTRA_OPTS=-Ddebug.addDefaultUser=true" \ - dremio/dremio-oss - -echo "Dremio service started." \ No newline at end of file diff --git a/.github/scripts/start_minio.sh b/.github/scripts/start_minio.sh deleted file mode 100644 index 2679edc1..00000000 --- a/.github/scripts/start_minio.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e - -: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}" -: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}" - -echo "Starting MinIO service..." - -docker run -d \ - --network ci-network \ - --name minio \ - -p 9000:9000 \ - -p 9001:9001 \ - -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ - -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ - minio/minio server /data --console-address ":9001" - -echo "MinIO service started." \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18184000..c7723649 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: - name: Extract Actual Failed Tests run: | shopt -s globstar - grep "FAILED tests" reports/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt + grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt - name: Sort Expected Failures run: sort .github/expected_failures.txt > expected_failures_sorted.txt From baa5b8c87e2da7ec73781f80a5f952161ebbe8f6 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Wed, 12 Mar 2025 17:19:18 +0000 Subject: [PATCH 4/5] Simplify failure comparison --- .github/workflows/ci.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7723649..83a1911e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,28 +131,24 @@ jobs: with: path: reports/ - - name: Extract Actual Failed Tests + - name: Compare Actual Failures with Expected Failures run: | shopt -s globstar - grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt - - - name: Sort Expected Failures - run: sort .github/expected_failures.txt > expected_failures_sorted.txt + actual_failures=$(grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort) + expected_failures=$(sort .github/expected_failures.txt) - - name: Compare Actual Failures with Expected Failures - run: | echo "Expected Failures:" - cat expected_failures_sorted.txt + echo "$expected_failures" echo "" echo "Actual Failures:" - cat actual_failures_sorted.txt + echo "$actual_failures" echo "" # Identify unexpected failures - unexpected_failures=$(comm -13 expected_failures_sorted.txt actual_failures_sorted.txt) + unexpected_failures=$(comm -13 <(echo "$expected_failures") <(echo "$actual_failures")) # Identify missing expected failures - missing_failures=$(comm -23 expected_failures_sorted.txt actual_failures_sorted.txt) + missing_failures=$(comm -23 <(echo "$expected_failures") <(echo "$actual_failures")) # Initialize exit code exit_code=0 @@ -175,4 +171,4 @@ jobs: echo "Verification failed: There are unexpected or missing test failures." fi - exit $exit_code \ No newline at end of file + exit $exit_code From 881e5f803f6214903448bd15d2c95da2c4e973b5 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Fri, 14 Mar 2025 10:10:29 +0000 Subject: [PATCH 5/5] Add initally removed script files --- .github/scripts/run_tests.sh | 25 +++++++++++++++++++++++++ .github/scripts/start_dremio.sh | 14 ++++++++++++++ .github/scripts/start_minio.sh | 18 ++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 .github/scripts/run_tests.sh create mode 100644 .github/scripts/start_dremio.sh create mode 100644 .github/scripts/start_minio.sh diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh new file mode 100644 index 00000000..9b2282a7 --- /dev/null +++ b/.github/scripts/run_tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +mkdir -p reports + +echo "Starting tests..." + +test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u) + +echo "Test directories found:" +echo "$test_dirs" + +# Run tests in each directory and save reports +for dir in $test_dirs; do + echo "Running tests in directory: $dir" + + # Generate a safe report filename + report_file="reports/$(echo "$dir" | tr '/' '_').txt" + + echo "Saving report to: $report_file" + + pytest "$dir" | tee "$report_file" +done + +echo "All tests executed successfully." \ No newline at end of file diff --git a/.github/scripts/start_dremio.sh b/.github/scripts/start_dremio.sh new file mode 100644 index 00000000..972c97c0 --- /dev/null +++ b/.github/scripts/start_dremio.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +echo "Starting Dremio service..." + +docker run -d \ + --network ci-network \ + --name dremio \ + -p 31010:31010 \ + -p 9047:9047 \ + -e "DREMIO_JAVA_SERVER_EXTRA_OPTS=-Ddebug.addDefaultUser=true" \ + dremio/dremio-oss + +echo "Dremio service started." \ No newline at end of file diff --git a/.github/scripts/start_minio.sh b/.github/scripts/start_minio.sh new file mode 100644 index 00000000..2679edc1 --- /dev/null +++ b/.github/scripts/start_minio.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}" +: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}" + +echo "Starting MinIO service..." + +docker run -d \ + --network ci-network \ + --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ + -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ + minio/minio server /data --console-address ":9001" + +echo "MinIO service started." \ No newline at end of file