feat: CI with AWS S3 access (2) #84
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| # runtime is erratic and up to an hour | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| permissions: # Add this at the workflow level | |
| id-token: write | |
| contents: read | |
| pull-requests: read # Add this for PR events | |
| jobs: | |
| unit-tests: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 # avoids ever triggering a rate limit | |
| matrix: | |
| python-version: ['3.11'] | |
| os: [ubuntu-latest] | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Debug OIDC setup | |
| run: | | |
| echo "=== GitHub Context ===" | |
| echo "Repository: ${{ github.repository }}" | |
| echo "Ref: ${{ github.ref }}" | |
| echo "Event: ${{ github.event_name }}" | |
| echo "=== Environment Variables ===" | |
| echo "ACTIONS_ID_TOKEN_REQUEST_URL: $ACTIONS_ID_TOKEN_REQUEST_URL" | |
| echo "ACTIONS_ID_TOKEN_REQUEST_TOKEN: $ACTIONS_ID_TOKEN_REQUEST_TOKEN" | |
| if [ -z "$ACTIONS_ID_TOKEN_REQUEST_URL" ]; then | |
| echo "ERROR: ACTIONS_ID_TOKEN_REQUEST_URL is not set!" | |
| echo "This means GitHub Actions doesn't have permission to generate OIDC tokens" | |
| fi | |
| - name: Configure AWS credentials from OIDC | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::837454214164:role/GitHubActions-Role | |
| aws-region: us-east-1 | |
| - name: Verify AWS identity (optional) | |
| run: | | |
| aws sts get-caller-identity | |
| - name: Upload files to S3 | |
| run: | | |
| # Example: ls ci bucket | |
| aws s3 ls s3://commoncrawl-ci-temp/ | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install setuptools on python 3.12+ | |
| if: ${{ matrix.python-version >= '3.12' }} | |
| run: | | |
| pip install setuptools | |
| - name: Install cdx_toolkit | |
| run: pip install .[test] | |
| - name: Run AWS tests | |
| run: | | |
| python aws_test.py | |
| - name: Run tests | |
| run: | | |
| make test_coverage | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} |