Skip to content

Commit 6388e32

Browse files
dbeatty10dbeatty10
authored
Independent workflow job for dbt-postgres (#215)
* Independent workflow job for dbt-postgres * Remove activation of virtual environment * Try without `python -m` * Independent workflow job for dbt-redshift * Independent workflow job for dbt-snowflake * Independent workflow job for dbt-snowflake * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Independent workflow job for dbt-bigquery * Setup environment variables for dbt-bigquery
1 parent 5caf2e3 commit 6388e32

File tree

1 file changed

+76
-47
lines changed

1 file changed

+76
-47
lines changed

‎.circleci/config.yml‎

Lines changed: 76 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,100 +1,129 @@
11
version: 2
22

33
jobs:
4-
build:
4+
5+
integration-postgres:
56
docker:
6-
- image: cimg/python:3.9.9
7+
- image: cimg/python:3.9
78
- image: cimg/postgres:9.6
89
environment:
910
POSTGRES_USER: root
11+
environment:
12+
POSTGRES_HOST: localhost
13+
POSTGRES_USER: root
14+
DBT_ENV_SECRET_POSTGRES_PASS: ''
15+
POSTGRES_PORT: 5432
16+
POSTGRES_DATABASE: circle_test
17+
POSTGRES_SCHEMA: codegen_integration_tests_postgres
18+
1019
steps:
1120
- checkout
12-
13-
- run:
14-
name: setup_creds
15-
command: |
16-
echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
17-
18-
- restore_cache:
19-
key: deps1-{{ .Branch }}
20-
21-
- run:
22-
name: "Setup dbt"
23-
command: |
24-
python3 -m venv dbt_venv
25-
. dbt_venv/bin/activate
26-
27-
python -m pip install --upgrade pip setuptools
28-
python -m pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery
29-
21+
- run: pip install --pre dbt-core dbt-postgres
3022
- run:
3123
name: "Run Tests - Postgres"
32-
environment:
33-
POSTGRES_HOST: localhost
34-
POSTGRES_USER: root
35-
DBT_ENV_SECRET_POSTGRES_PASS: ""
36-
POSTGRES_PORT: 5432
37-
POSTGRES_DATABASE: circle_test
38-
POSTGRES_SCHEMA: codegen_integration_tests_postgres
3924
command: |
40-
. dbt_venv/bin/activate
4125
cd integration_tests
4226
dbt --warn-error deps --target postgres
4327
dbt --warn-error run-operation create_source_table --target postgres
4428
dbt --warn-error seed --target postgres --full-refresh
4529
dbt --warn-error run --target postgres
4630
dbt --warn-error test --target postgres
31+
- store_artifacts:
32+
path: integration_tests/logs
33+
- store_artifacts:
34+
path: integration_tests/target
35+
36+
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
37+
resource_class: large
4738

39+
integration-redshift:
40+
docker:
41+
- image: cimg/python:3.9
42+
steps:
43+
- checkout
44+
- run: pip install --pre dbt-core dbt-redshift
4845
- run:
4946
name: "Run Tests - Redshift"
5047
command: |
51-
. dbt_venv/bin/activate
52-
echo `pwd`
5348
cd integration_tests
5449
dbt --warn-error deps --target redshift
5550
dbt --warn-error run-operation create_source_table --target redshift
5651
dbt --warn-error seed --target redshift --full-refresh
5752
dbt --warn-error run --target redshift
5853
dbt --warn-error test --target redshift
54+
- store_artifacts:
55+
path: integration_tests/logs
56+
- store_artifacts:
57+
path: integration_tests/target
58+
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
59+
resource_class: large
5960

61+
integration-snowflake:
62+
docker:
63+
- image: cimg/python:3.9
64+
steps:
65+
- checkout
66+
- run: pip install --pre dbt-core dbt-snowflake
6067
- run:
6168
name: "Run Tests - Snowflake"
6269
command: |
63-
. dbt_venv/bin/activate
64-
echo `pwd`
6570
cd integration_tests
6671
dbt --warn-error deps --target snowflake
6772
dbt --warn-error run-operation create_source_table --target snowflake
6873
dbt --warn-error seed --target snowflake --full-refresh
6974
dbt --warn-error run --target snowflake
7075
dbt --warn-error test --target snowflake
76+
- store_artifacts:
77+
path: integration_tests/logs
78+
- store_artifacts:
79+
path: integration_tests/target
80+
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
81+
resource_class: large
7182

83+
integration-bigquery:
84+
environment:
85+
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
86+
docker:
87+
- image: cimg/python:3.9
88+
steps:
89+
- checkout
90+
- run: pip install --pre dbt-core dbt-bigquery
91+
- run:
92+
name: Setup Environment Variables
93+
command: |
94+
echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
95+
echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV"
7296
- run:
7397
name: "Run Tests - BigQuery"
74-
environment:
75-
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
76-
7798
command: |
78-
. dbt_venv/bin/activate
79-
echo `pwd`
8099
cd integration_tests
81100
dbt --warn-error deps --target bigquery
82101
dbt --warn-error run-operation create_source_table --target bigquery
83102
dbt --warn-error seed --target bigquery --full-refresh
84103
dbt --warn-error run --target bigquery
85104
dbt --warn-error test --target bigquery
86-
87-
- save_cache:
88-
key: deps1-{{ .Branch }}
89-
paths:
90-
- "dbt_venv"
105+
- store_artifacts:
106+
path: integration_tests/logs
107+
- store_artifacts:
108+
path: integration_tests/target
109+
# The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
110+
resource_class: large
91111

92112
workflows:
93113
version: 2
94114
test-all:
95115
jobs:
96-
- build:
97-
context:
98-
- profile-redshift
99-
- profile-snowflake
100-
- profile-bigquery
116+
- integration-postgres:
117+
context: profile-postgres
118+
- integration-redshift:
119+
context: profile-redshift
120+
requires:
121+
- integration-postgres
122+
- integration-snowflake:
123+
context: profile-snowflake
124+
requires:
125+
- integration-postgres
126+
- integration-bigquery:
127+
context: profile-bigquery
128+
requires:
129+
- integration-postgres

0 commit comments

Comments
 (0)