Skip to content

Commit cd81768

Browse files
dolfinusdolfinus
committed
Merge branch 'develop'
2 parents d570433 + 23f4f44 commit cd81768

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+493
-677
lines changed

‎.env.docker‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,4 @@ ENV=LOCAL
5151
SPARK_CONF_DIR=/app/tests/spark/hive/conf/
5252
HADOOP_CONF_DIR=/app/tests/spark/hadoop/
5353
HIVE_CONF_DIR=/app/tests/spark/hive/conf/
54+
CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session.get_worker_spark_session

‎.env.local‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,5 @@ export ENV=LOCAL
5050

5151
export SPARK_CONF_DIR=./app/tests/spark/hive/conf/
5252
export HADOOP_CONF_DIR=./app/tests/spark/hadoop/
53-
export HIVE_CONF_DIR=./app/tests/spark/hive/conf/
53+
export HIVE_CONF_DIR=./app/tests/spark/hive/conf/
54+
export CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session.get_worker_spark_session

‎.github/workflows/backend_docker_image.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
with:
4949
tags: ${{ env.TAG }}
5050
context: .
51-
file: docker/backend.dockerfile
51+
file: docker/Dockerfile.backend
5252
pull: true
5353
push: true
5454
cache-to: type=gha,mode=max

‎.github/workflows/hdfs-tests.yml‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ jobs:
4747
uses: docker/build-push-action@v5
4848
with:
4949
context: .
50-
tags: syncmaster_worker:${{ github.sha }}
50+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
5151
target: test
52-
file: docker/worker.dockerfile
52+
file: docker/Dockerfile.worker
5353
load: true
5454
cache-to: type=gha,mode=max
5555
cache-from: type=gha
@@ -60,22 +60,16 @@ jobs:
6060
docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-hive test-postgres --wait --wait-timeout 200
6161
env:
6262
WORKER_IMAGE_TAG: ${{ github.sha }}
63-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
6463

65-
- name: Run HDFS Tests.
66-
# Tests for the backend are run on the worker.
67-
# Backend and worker on the same container.
64+
# This is important, as coverage is exported after receiving SIGTERM
65+
- name: Run HDFS Tests
6866
run: |
69-
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m hdfs
70-
env:
71-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
67+
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and hdfs"
7268
7369
- name: Shutdown
7470
if: always()
7571
run: |
7672
docker compose -f docker-compose.test.yml down -v --remove-orphans
77-
env:
78-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
7973
8074
- name: Upload coverage results
8175
uses: actions/upload-artifact@v4

‎.github/workflows/hive-tests.yml‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ jobs:
4747
uses: docker/build-push-action@v5
4848
with:
4949
context: .
50-
tags: syncmaster_worker:${{ github.sha }}
50+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
5151
target: test
52-
file: docker/worker.dockerfile
52+
file: docker/Dockerfile.worker
5353
load: true
5454
cache-to: type=gha,mode=max
5555
cache-from: type=gha
@@ -60,22 +60,16 @@ jobs:
6060
docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-hive test-postgres --wait --wait-timeout 200
6161
env:
6262
WORKER_IMAGE_TAG: ${{ github.sha }}
63-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
6463

65-
- name: Run Hive Tests.
66-
# Tests for the backend are run on the worker.
67-
# Backend and worker on the same container.
64+
# This is important, as coverage is exported after receiving SIGTERM
65+
- name: Run Hive Tests
6866
run: |
69-
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m hive
70-
env:
71-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
67+
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and hive"
7268
7369
- name: Shutdown
7470
if: always()
7571
run: |
7672
docker compose -f docker-compose.test.yml down -v --remove-orphans
77-
env:
78-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
7973
8074
- name: Upload coverage results
8175
uses: actions/upload-artifact@v4

‎.github/workflows/oracle-tests.yml‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ jobs:
4747
uses: docker/build-push-action@v5
4848
with:
4949
context: .
50-
tags: syncmaster_worker:${{ github.sha }}
50+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
5151
target: test
52-
file: docker/worker.dockerfile
52+
file: docker/Dockerfile.worker
5353
load: true
5454
cache-to: type=gha,mode=max
5555
cache-from: type=gha
@@ -60,22 +60,16 @@ jobs:
6060
docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-oracle test-postgres --wait --wait-timeout 200
6161
env:
6262
WORKER_IMAGE_TAG: ${{ github.sha }}
63-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
6463

65-
- name: Run Oracle Tests.
66-
# Tests for the backend are run on the worker.
67-
# Backend and worker on the same container.
64+
# This is important, as coverage is exported after receiving SIGTERM
65+
- name: Run Oracle Tests
6866
run: |
69-
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m oracle
70-
env:
71-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
67+
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and oracle"
7268
7369
- name: Shutdown
7470
if: always()
7571
run: |
7672
docker compose -f docker-compose.test.yml down -v --remove-orphans
77-
env:
78-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
7973
8074
- name: Upload coverage results
8175
uses: actions/upload-artifact@v4

‎.github/workflows/s3-tests.yml‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ jobs:
4747
uses: docker/build-push-action@v5
4848
with:
4949
context: .
50-
tags: syncmaster_worker:${{ github.sha }}
51-
file: docker/worker.dockerfile
50+
tags: mtsrus/syncmaster-worker:${{ github.sha }}
51+
file: docker/Dockerfile.worker
5252
target: test
5353
load: true
5454
cache-to: type=gha,mode=max
@@ -60,22 +60,16 @@ jobs:
6060
docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-s3 test-postgres --wait --wait-timeout 200
6161
env:
6262
WORKER_IMAGE_TAG: ${{ github.sha }}
63-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
6463

65-
- name: Run S3 Tests.
66-
# Tests for the backend are run on the worker.
67-
# Backend and worker on the same container.
64+
- name: Run S3 Tests
6865
run: |
69-
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m s3
70-
env:
71-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
66+
docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and s3"
7267
68+
# This is important, as coverage is exported after receiving SIGTERM
7369
- name: Shutdown
7470
if: always()
7571
run: |
7672
docker compose -f docker-compose.test.yml down -v --remove-orphans
77-
env:
78-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
7973
8074
- name: Upload coverage results
8175
uses: actions/upload-artifact@v4

‎.github/workflows/unit-test.yml‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,22 +52,17 @@ jobs:
5252
run: |
5353
docker compose -f docker-compose.test.yml down -v --remove-orphans
5454
docker compose -f docker-compose.test.yml up -d db --wait --wait-timeout 200
55-
env:
56-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
5755
58-
- name: Run Unit Tests.
56+
- name: Run Unit Tests
5957
run: |
6058
source .env.local
61-
./pytest_runner.sh -m backend
62-
env:
63-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
59+
poetry run python -m syncmaster.db.migrations upgrade head
60+
poetry run coverage run -m pytest -vvv -s -m backend
6461
6562
- name: Shutdown
6663
if: always()
6764
run: |
6865
docker compose -f docker-compose.test.yml down -v --remove-orphans
69-
env:
70-
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
7166
7267
- name: Upload coverage results
7368
uses: actions/upload-artifact@v4

‎.github/workflows/worker_docker_image.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
tags: ${{ env.TAG }}
5050
context: .
5151
target: prod
52-
file: docker/worker.dockerfile
52+
file: docker/Dockerfile.worker
5353
pull: true
5454
push: true
5555
cache-to: type=gha,mode=max

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,3 +158,5 @@ cython_debug/
158158
**/metastore_db/
159159
**/spark-warehouse/
160160
cached_jars/
161+
162+
.DS_Store

0 commit comments

Comments
 (0)