Daily dbt Sync #76
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily dbt Sync | |
| on: | |
| schedule: | |
| - cron: '0 8 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| sync_artifacts: | |
| name: Sync dbt Artifacts from S3 | |
| runs-on: datahub-runner | |
| steps: | |
| - name: Sync dbt Artifacts | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: eu-west-1 | |
| run: | | |
| set -eu | |
| echo "📂 Syncing dbt artifacts..." | |
| S3_BUCKET="s3://data.studocu.com/target/" | |
| LOCAL_DIR="ingestion/dbt_artifacts" | |
| # Create Local Directory if It Does Not Exist | |
| if [ ! -d "$LOCAL_DIR" ]; then | |
| echo "Local directory $LOCAL_DIR does not exist. Creating it..." | |
| mkdir -p "$LOCAL_DIR" | |
| fi | |
| # Sync the dbt Artifacts from S3 | |
| echo "Starting sync from $S3_BUCKET to $LOCAL_DIR ..." | |
| aws s3 sync "$S3_BUCKET" "$LOCAL_DIR" --delete | |
| echo "✅ Sync complete." | |
| working-directory: /home/ubuntu/datahub |