Skip to content

Commit e20e92c

Browse files
99Lys99Lys
authored
Refactor CI scripts and test fixes (#304)
### Summary Refactoring CI scripts and fixing some tests. ### Description [CI Refactoring] - Deleted `create_dbt_test_users.sh` → users are only needed for grants tests, which are not being ran in the current CI - Created `extract_auth_token.sh` and updated `create_and_format_sources.sh` → split authentication token extraction from sources creation - Updated `create_env_file.sh` → removed users and roles, as they are only needed for grants tests - Display the exit status of each test/job → this gives us more visibility on what group of tests failed exactly, unfortunately the pipeline as a whole will be considered as "failed", but we only need to consider the results of **Verify Expected Test Failures** job - Fixed erroring tests not being considered as failing tests → look for `ERROR tests` in artifacts as well, not only `FAILED tests` [Tests Fixing] - Grants → override `apply_grants` macro to include `user:` if no prefix is defined, otherwise any rerun of the same model will lead to revoking and regranting; override tests to include prefix in the expected results - Hooks → add test prefix in dataset folder creation to avoid existing folder error ### Test Results - All tests are passing/failing as expected ### Changelog - [x] Added a summary of what this PR accomplishes to CHANGELOG.md
1 parent 2258844 commit e20e92c

File tree

15 files changed

+357
-128
lines changed

15 files changed

+357
-128
lines changed

‎.github/scripts/compare_test_failures.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "Comparing actual test failures with expected failures..."
5+
6+
# Enable globstar for recursive globbing
7+
shopt -s globstar
8+
9+
# Extract actual failures from test reports
10+
actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort)
11+
12+
# Read expected failures
13+
expected_failures=$(sort .github/expected_failures.txt)
14+
15+
echo "Expected Failures:"
16+
echo "$expected_failures"
17+
echo ""
18+
echo "Actual Failures:"
19+
echo "$actual_failures"
20+
echo ""
21+
22+
# Identify unexpected failures (in actual but not in expected)
23+
unexpected_failures=$(comm -13 <(echo "$expected_failures") <(echo "$actual_failures"))
24+
25+
# Identify missing expected failures (in expected but not in actual)
26+
missing_failures=$(comm -23 <(echo "$expected_failures") <(echo "$actual_failures"))
27+
28+
# Initialize exit code
29+
exit_code=0
30+
31+
if [ -n "$unexpected_failures" ]; then
32+
echo "Unexpected test failures detected:"
33+
echo "$unexpected_failures"
34+
exit_code=1
35+
fi
36+
37+
if [ -n "$missing_failures" ]; then
38+
echo "::warning::Expected test failures that did not occur (they passed):"
39+
echo "$missing_failures"
40+
exit_code=1
41+
fi
42+
43+
if [ $exit_code -eq 0 ]; then
44+
echo "All failed tests are expected, and all expected failures have occurred."
45+
else
46+
echo "Verification failed: There are unexpected or missing test failures."
47+
fi
48+
49+
exit $exit_code

‎.github/scripts/create_and_format_sources.sh‎

Lines changed: 7 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,20 @@
11
#!/bin/bash
22
set -e
33

4-
: "${RETRY_COUNT:?Need to set RETRY_COUNT}"
54
: "${DREMIO_HEALTH_URL:?Need to set DREMIO_HEALTH_URL}"
6-
: "${SLEEP_INTERVAL:?Need to set SLEEP_INTERVAL}"
7-
: "${DREMIO_SOFTWARE_USERNAME:?Need to set DREMIO_SOFTWARE_USERNAME}"
8-
: "${DREMIO_SOFTWARE_PASSWORD:?Need to set DREMIO_SOFTWARE_PASSWORD}"
95
: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}"
106
: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}"
117

12-
for i in $(seq 1 $RETRY_COUNT); do
13-
if curl -s $DREMIO_HEALTH_URL; then
14-
echo "Dremio is up."
15-
break
16-
fi
17-
echo "Waiting for Dremio to become ready... ($i/$RETRY_COUNT)"
18-
sleep $SLEEP_INTERVAL
19-
done
20-
21-
if ! curl -s $DREMIO_HEALTH_URL; then
22-
echo "Dremio did not become ready in time."
23-
exit 1
24-
fi
25-
26-
# Obtain Dremio auth token
27-
echo "Logging into Dremio to obtain auth token..."
28-
AUTH_RESPONSE=$(curl -s -X POST "$DREMIO_HEALTH_URL/apiv2/login" \
29-
-H "Content-Type: application/json" \
30-
--data "{\"userName\":\"${DREMIO_SOFTWARE_USERNAME}\", \"password\":\"${DREMIO_SOFTWARE_PASSWORD}\"}")
31-
32-
AUTH_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r .token)
33-
34-
# Check if AUTH_TOKEN is not empty
35-
if [ -z "$AUTH_TOKEN" ] || [ "$AUTH_TOKEN" == "null" ]; then
36-
echo "Failed to obtain Dremio auth token."
37-
exit 1
38-
fi
39-
40-
echo "Obtained Dremio auth token."
41-
echo "::add-mask::$AUTH_TOKEN"
8+
# Get AUTH_TOKEN from environment or file
429
if [ "$GITHUB_ACTIONS" = "true" ]; then
43-
echo "Running in GitHub Actions"
44-
echo "AUTH_TOKEN=${AUTH_TOKEN}" >> $GITHUB_ENV
10+
: "${AUTH_TOKEN:?Need to set AUTH_TOKEN}"
4511
HOST="minio"
4612
else # Jenkins
47-
echo $AUTH_TOKEN > /tmp/auth_token.txt
13+
if [ ! -f /tmp/auth_token.txt ]; then
14+
echo "Auth token file not found. Please run extract_auth_token.sh first."
15+
exit 1
16+
fi
17+
AUTH_TOKEN=$(cat /tmp/auth_token.txt)
4818
HOST="localhost"
4919
fi
5020

‎.github/scripts/create_dbt_test_users.sh‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎.github/scripts/create_env_file.sh‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,7 @@ DREMIO_SOFTWARE_USERNAME=${DREMIO_SOFTWARE_USERNAME}
1313
DREMIO_SOFTWARE_PASSWORD=${DREMIO_SOFTWARE_PASSWORD}
1414
DREMIO_DATALAKE=dbt_test_source
1515
DREMIO_DATABASE=dbt_test
16-
DBT_TEST_USER_1=dbt_test_user_1
17-
DBT_TEST_USER_2=dbt_test_user_2
18-
DBT_TEST_USER_3=dbt_test_user_3
19-
DBT_TEST_ROLE_1=dbt_test_role_1
20-
DBT_TEST_ROLE_2=dbt_test_role_2
2116
DREMIO_EDITION=community
2217
EOF
2318

24-
echo ".env file created successfully."
19+
echo ".env file created successfully."

‎.github/scripts/extract_auth_token.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
set -e
3+
4+
: "${RETRY_COUNT:?Need to set RETRY_COUNT}"
5+
: "${DREMIO_HEALTH_URL:?Need to set DREMIO_HEALTH_URL}"
6+
: "${SLEEP_INTERVAL:?Need to set SLEEP_INTERVAL}"
7+
: "${DREMIO_SOFTWARE_USERNAME:?Need to set DREMIO_SOFTWARE_USERNAME}"
8+
: "${DREMIO_SOFTWARE_PASSWORD:?Need to set DREMIO_SOFTWARE_PASSWORD}"
9+
10+
for i in $(seq 1 $RETRY_COUNT); do
11+
if curl -s $DREMIO_HEALTH_URL; then
12+
echo "Dremio is up."
13+
break
14+
fi
15+
echo "Waiting for Dremio to become ready... ($i/$RETRY_COUNT)"
16+
sleep $SLEEP_INTERVAL
17+
done
18+
19+
if ! curl -s $DREMIO_HEALTH_URL; then
20+
echo "Dremio did not become ready in time."
21+
exit 1
22+
fi
23+
24+
# Obtain Dremio auth token
25+
echo "Logging into Dremio to obtain auth token..."
26+
AUTH_RESPONSE=$(curl -s -X POST "$DREMIO_HEALTH_URL/apiv2/login" \
27+
-H "Content-Type: application/json" \
28+
--data "{\"userName\":\"${DREMIO_SOFTWARE_USERNAME}\", \"password\":\"${DREMIO_SOFTWARE_PASSWORD}\"}")
29+
30+
AUTH_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r .token)
31+
32+
# Check if AUTH_TOKEN is not empty
33+
if [ -z "$AUTH_TOKEN" ] || [ "$AUTH_TOKEN" == "null" ]; then
34+
echo "Failed to obtain Dremio auth token."
35+
exit 1
36+
fi
37+
38+
echo "Obtained Dremio auth token."
39+
echo "::add-mask::$AUTH_TOKEN"
40+
if [ "$GITHUB_ACTIONS" = "true" ]; then
41+
echo "AUTH_TOKEN=${AUTH_TOKEN}" >> $GITHUB_ENV
42+
else # Jenkins
43+
echo $AUTH_TOKEN > /tmp/auth_token.txt
44+
fi

‎.github/workflows/ci.yml‎

Lines changed: 9 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ jobs:
2727
run-parallel-tests:
2828
name: Run Parallel Tests
2929
runs-on: ubuntu-latest
30-
needs: discover-tests
30+
continue-on-error: true
31+
needs: discover-tests
3132
env:
3233
RETRY_COUNT: 12 # number of retries for health checks
3334
SLEEP_INTERVAL: 5 # Sleep duration in seconds between retries
@@ -75,6 +76,9 @@ jobs:
7576
- name: Create MinIO bucket
7677
run: bash .github/scripts/create_minio_bucket.sh
7778

79+
- name: Extract Auth Token
80+
run: bash .github/scripts/extract_auth_token.sh
81+
7882
- name: Create and Format Sources
7983
run: bash .github/scripts/create_and_format_sources.sh
8084

@@ -89,9 +93,6 @@ jobs:
8993
pip install -r dev_requirements.txt
9094
pip install .
9195
92-
- name: Create dbt test users
93-
run: bash .github/scripts/create_dbt_test_users.sh
94-
9596
- name: Create dbt projects
9697
run: bash .github/scripts/create_dbt_projects.sh
9798

@@ -107,13 +108,15 @@ jobs:
107108
run: echo "sanitized_test_dir=$(echo ${{ matrix.test_dir }} | tr '/' '_')" >> $GITHUB_ENV
108109

109110
- name: Run tests
110-
continue-on-error: true
111+
id: run_tests
111112
run: |
112113
mkdir reports/
113114
report_file="reports/${{ env.sanitized_test_dir }}.txt"
114115
pytest ${{ matrix.test_dir }} | tee $report_file
116+
exit ${PIPESTATUS[0]}
115117
116118
- name: Upload test report as artifact
119+
if: always()
117120
uses: actions/upload-artifact@v4
118121
with:
119122
name: ${{ env.sanitized_test_dir }}
@@ -133,43 +136,4 @@ jobs:
133136
path: reports/
134137

135138
- name: Compare Actual Failures with Expected Failures
136-
run: |
137-
shopt -s globstar
138-
actual_failures=$(grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort)
139-
expected_failures=$(sort .github/expected_failures.txt)
140-
141-
echo "Expected Failures:"
142-
echo "$expected_failures"
143-
echo ""
144-
echo "Actual Failures:"
145-
echo "$actual_failures"
146-
echo ""
147-
148-
# Identify unexpected failures
149-
unexpected_failures=$(comm -13 <(echo "$expected_failures") <(echo "$actual_failures"))
150-
151-
# Identify missing expected failures
152-
missing_failures=$(comm -23 <(echo "$expected_failures") <(echo "$actual_failures"))
153-
154-
# Initialize exit code
155-
exit_code=0
156-
157-
if [ -n "$unexpected_failures" ]; then
158-
echo "Unexpected test failures detected:"
159-
echo "$unexpected_failures"
160-
exit_code=1
161-
fi
162-
163-
if [ -n "$missing_failures" ]; then
164-
echo "Expected test failures that did not occur (they passed):"
165-
echo "$missing_failures"
166-
exit_code=1
167-
fi
168-
169-
if [ $exit_code -eq 0 ]; then
170-
echo "All failed tests are expected, and all expected failures have occurred."
171-
else
172-
echo "Verification failed: There are unexpected or missing test failures."
173-
fi
174-
175-
exit $exit_code
139+
run: bash .github/scripts/compare_test_failures.sh

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
## Changes
44

55
- [#299](https://github.com/dremio/dbt-dremio/pull/299) Enhance persist_docs macro to wrap model and column metadata (including descriptions, tags and tests) into a Markdown wiki for Dremio.
6+
- Refactored CI
7+
- Fixed tests for hooks and grants
68

79
# dbt-dremio v1.9.0
810

‎dbt/adapters/dremio/impl.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ def standardize_grants_dict(self, grants_table: agate.Table) -> dict:
161161
for row in grants_table:
162162
# Just needed to change these two values to match Dremio cols
163163
grantee = row["grantee_id"]
164-
privilege = row["privilege"]
164+
privilege = row["privilege"].lower()
165165
grantee_type = row["grantee_type"]
166166

167167
if privilege in grants_dict.keys():

‎dbt/include/dremio/macros/adapters/apply_grants.sql‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,48 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
See the License for the specific language governing permissions and
1313
limitations under the License.*/
1414

15+
{% macro dremio__apply_grants(relation, grant_config, should_revoke=True) %}
16+
{#-- If grant_config is {} or None, this is a no-op --#}
17+
{% if grant_config %}
18+
{#-- ensure grant have a prefix, defaults to user if not provided --#}
19+
{% for privilege, grantees in grant_config.items() %}
20+
{% set updated_grantees = [] %}
21+
{% for grantee in grantees %}
22+
{% if ':' not in grantee %}
23+
{% do updated_grantees.append('user:' ~ grantee) %}
24+
{% else %}
25+
{% do updated_grantees.append(grantee) %}
26+
{% endif %}
27+
{% endfor %}
28+
{% do grant_config.update({privilege: updated_grantees}) %}
29+
{% endfor %}
30+
{% if should_revoke %}
31+
{#-- We think previous grants may have carried over --#}
32+
{#-- Show current grants and calculate diffs --#}
33+
{% set current_grants_table = run_query(get_show_grant_sql(relation)) %}
34+
{% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}
35+
{% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}
36+
{% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}
37+
{% if not (needs_granting or needs_revoking) %}
38+
{{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}
39+
{% endif %}
40+
{% else %}
41+
{#-- We don't think there's any chance of previous grants having carried over. --#}
42+
{#-- Jump straight to granting what the user has configured. --#}
43+
{% set needs_revoking = {} %}
44+
{% set needs_granting = grant_config %}
45+
{% endif %}
46+
{% if needs_granting or needs_revoking %}
47+
{% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}
48+
{% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}
49+
{% set dcl_statement_list = revoke_statement_list + grant_statement_list %}
50+
{% if dcl_statement_list %}
51+
{{ call_dcl_statements(dcl_statement_list) }}
52+
{% endif %}
53+
{% endif %}
54+
{% endif %}
55+
{% endmacro %}
56+
1557
{%- macro dremio__support_multiple_grantees_per_dcl_statement() -%}
1658
{{ return(False) }}
1759
{%- endmacro -%}

‎tests/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ To run our tests, a test environment must be set up.
1717
DBT_TEST_USER_1=dbt_test_user_1
1818
DBT_TEST_USER_2=dbt_test_user_2
1919
DBT_TEST_USER_3=dbt_test_user_3
20+
DBT_TEST_ROLE_1=dbt_test_role_1
21+
DBT_TEST_ROLE_2=dbt_test_role_2
2022
```
2123
For cloud version of Dremio:
2224
```
@@ -29,6 +31,8 @@ To run our tests, a test environment must be set up.
2931
DBT_TEST_USER_1=dbt_test_user_1
3032
DBT_TEST_USER_2=dbt_test_user_2
3133
DBT_TEST_USER_3=dbt_test_user_3
34+
DBT_TEST_ROLE_1=dbt_test_role_1
35+
DBT_TEST_ROLE_2=dbt_test_role_2
3236
```
3337
1. Create the three users listed above (dbt_test_user_1, dbt_test_user_2, dbt_test_user_3) in the Dremio instance.
3438
1. Create a bucket in Object storage with a name `dbtdremios3`

0 commit comments

Comments
 (0)