Process and Release Service Data #342
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Process and Release Service Data | |
| on: | |
| push: | |
| branches: | |
| - master # Run on all pushes to master | |
| pull_request: | |
| branches: | |
| - master # Run on all PRs targeting master | |
| schedule: | |
| - cron: '0 10 * * 1-5' # daily runs monday to friday at 10:00 UTC | |
| workflow_dispatch: # Allow manual triggers | |
| jobs: | |
| generate-files: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # Needed for creating releases | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install requirements for python | |
| run: pip install -r requirements.txt | |
| - name: Get current date and commit hash | |
| id: metadata | |
| run: | | |
| echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| echo "time=$(date +'%H:%M')" >> $GITHUB_OUTPUT | |
| - name: Run main.py | |
| run: python main.py | |
| - name: Prepare live files for release | |
| id: prepare_live_files | |
| run: | | |
| mkdir -p live_data_release | |
| echo "Copying live data files (.csv and .log)..." | |
| find outputs inputs -type f \( -name "*.csv" -o -name "*.log" \) \ | |
| ! -path "*/snapshots/*" -size +0c \ | |
| -exec cp {} live_data_release/ \; -exec echo "Copied: {}" \; | |
| echo "live_data_dir=live_data_release" >> $GITHUB_ENV | |
| - name: Prepare 2025-03-01 files for Release | |
| id: prepare_files_2025-03-01 | |
| run: | | |
| mkdir -p snapshots_release/snapshot_2025-03-01 | |
| echo "Copying snapshot data files for 2025-03-01..." | |
| find outputs/snapshots/2025-03-01 inputs/snapshots/2025-03-01 -type f -name "*.csv" \ | |
| -exec cp {} snapshots_release/snapshot_2025-03-01/ \; \ | |
| -exec echo "Copied: {}" \; | |
| for file in snapshots_release/snapshot_2025-03-01/*.csv; do | |
| mv "$file" "snapshots_release/2025-03-01_$(basename "$file")" | |
| done | |
| echo "snapshot_dir=snapshots_release" >> $GITHUB_ENV | |
| - name: Prepare 2026-01-12 files for Release | |
| id: prepare_files_2026-01-12 | |
| run: | | |
| mkdir -p snapshots_release/snapshot_2026-01-12 | |
| echo "Copying snapshot data files for 2026-01-12..." | |
| find outputs/snapshots/2026-01-12 inputs/snapshots/2026-01-12 -type f -name "*.csv" \ | |
| -exec cp {} snapshots_release/snapshot_2026-01-12/ \; \ | |
| -exec echo "Copied: {}" \; | |
| for file in snapshots_release/snapshot_2026-01-12/*.csv; do | |
| mv "$file" "snapshots_release/2026-01-12_$(basename "$file")" | |
| done | |
| echo "snapshot_dir=snapshots_release" >> $GITHUB_ENV | |
| - name: Create Release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| tag_name: service_data-${{ steps.metadata.outputs.commit_hash }} | |
| name: Service Data - ${{ steps.metadata.outputs.date }} | |
| body: | | |
| Automated service data release for ${{ steps.metadata.outputs.date }} | |
| - Generated on: ${{ steps.metadata.outputs.date }} at ${{ steps.metadata.outputs.time }} EST | |
| - Git Commit: ${{ steps.metadata.outputs.commit_hash }} | |
| Contains: | |
| - Live data CSVs | |
| - Snapshots pre-pended with date of snapshot | |
| files: | | |
| live_data_release/** | |
| snapshots_release/** | |
| make_latest: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Cleanup | |
| - name: Cleanup | |
| run: rm -rf outputs inputs snapshots_release live_data_release | |