dag-updated #28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Submodules on Dispatch | |
| on: | |
| repository_dispatch: | |
| types: [dag-updated] | |
| workflow_dispatch: | |
| jobs: | |
| update-submodules: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout umbrella repo | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| token: ${{ secrets.SUBMODULE_PAT }} | |
| fetch-depth: 1 | |
| - name: Update all submodules | |
| run: | | |
| git submodule update --remote --recursive | |
| # Check out main branch in each submodule | |
| for d in $(git config --file .gitmodules --get-regexp path | awk '{ print $2 }'); do | |
| echo "Checking out main branch in $d" | |
| cd "$d" | |
| git checkout main | |
| git pull origin main | |
| cd - | |
| done | |
| - name: Gather DAGs | |
| run: | | |
| rm -rf old_dags && mv dags old_dags || true | |
| rm -rf tmp dags | |
| mkdir -p tmp | |
| git submodule foreach --recursive ' | |
| echo "Processing $name ($path)..." | |
| if [ -d "$toplevel/$path/dags" ]; then | |
| mkdir -p "$toplevel/tmp/$name" | |
| cp -r "$toplevel/$path/dags/." "$toplevel/tmp/$name/" | |
| echo "Copied DAGs from $name:" | |
| ls -la "$toplevel/$path/dags" | |
| else | |
| echo "No dags/ directory found in $name" | |
| fi | |
| ' | |
| mv tmp dags | |
| - name: Push changes using fresh clone | |
| run: | | |
| WORKSPACE_PATH=$(pwd) | |
| mkdir -p /tmp/repo-push | |
| cd /tmp/repo-push | |
| git clone https://x-access-token:${{ secrets.CDR_PUSH_PAT }}@github.com/bcdev/airflow-dags.git . | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| rm -rf .git/modules/ | |
| cp -r $WORKSPACE_PATH/.git/modules/ .git/ | |
| cp -r $WORKSPACE_PATH/.gitmodules . | |
| rm -rf dags/ | |
| cp -r $WORKSPACE_PATH/dags ./dags | |
| for d in $(git config --file .gitmodules --get-regexp path | awk '{ print $2 }'); do | |
| echo "Copying updated submodule $d" | |
| rm -rf "$d" | |
| mkdir -p "$(dirname "$d")" | |
| cp -r "$WORKSPACE_PATH/$d" "$d" | |
| done | |
| git add . | |
| if ! git diff --cached --quiet; then | |
| git commit -m "update submodules and dags via dispatch" | |
| git push origin HEAD:main | |
| echo "Changes pushed successfully" | |
| else | |
| echo "No changes to commit." | |
| fi |