Scraping CMRL site #329
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scraping CMRL site | |
| on: | |
| schedule: | |
| - cron: "40 18 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| Scrape-cmrl-api: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: "Set up Python" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: "pyproject.toml" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - name: Install the project | |
| run: uv sync --all-extras --dev | |
| - name: "Run main.py" | |
| run: uv run main.py | |
| - name: Initiate Git | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Initialize date variable | |
| id: init-variable | |
| run: | | |
| export TZ=Asia/Kolkata | |
| CURRENT_DATE=$(date -d '-1 day' '+%d-%m-%Y') | |
| echo "CURRENT_DATE=$CURRENT_DATE" >> "$GITHUB_OUTPUT" | |
| - name: Commit changes | |
| run: | | |
| git add . | |
| git commit -m "Update scraped data on ${{steps.init-variable.outputs.CURRENT_DATE}} [skip ci]" | |
| git tag -a "v${{steps.init-variable.outputs.CURRENT_DATE}}" -m "Updated data from ${{steps.init-variable.outputs.CURRENT_DATE}}" | |
| git push origin | |
| git push origin --tags | |
| - name: Create GitHub Release using softprops/action-gh-release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| tag_name: "v${{steps.init-variable.outputs.CURRENT_DATE}}" # Use the tag we created | |
| name: "Scraped Data for ${{steps.init-variable.outputs.CURRENT_DATE}}" # Release title | |
| body: | | |
| ### Automated daily data release. | |
| This is an automatically generated daily release of scraped CSV data for ${{steps.init-variable.outputs.CURRENT_DATE}}. | |
| # Release description | |
| files: data/*.csv # Specify CSV files in the data folder as release assets | |
| draft: false | |
| prerelease: false |