Add password for SSH, AWS and GDrive config #2176
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| schedule: | |
| # Cron runs on the 1st and 15th of every month. | |
| # This will only run on main by default. | |
| - cron: "0 0 1,15 * *" | |
| jobs: | |
| linting: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: neuroinformatics-unit/actions/lint@v2 | |
| test: | |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false | |
| needs: [linting] | |
| name: ${{ matrix.os }} py${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| # macos-14 is M1, macos-13 is intel. Run on earliest and | |
| # latest python versions. All python versions are tested in | |
| # the weekly cron job. | |
| # Test all Python versions for cron job, and only first/last for other triggers | |
| os: ${{ fromJson(github.event_name == 'schedule' | |
| && '["ubuntu-latest","windows-latest","macos-14","macos-13"]' | |
| || '["ubuntu-latest","windows-latest","macos-latest"]') }} | |
| python-version: ${{ fromJson(github.event_name == 'schedule' | |
| && '["3.9","3.10","3.11","3.12","3.13"]' | |
| || '["3.9","3.13"]') }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| auto-update-conda: true | |
| channels: conda-forge | |
| activate-environment: "datashuttle-test" | |
| # The recommended installation is via conda, but we need to test | |
| # against dependencies from the pyproject.toml on the branch | |
| # to ensure dependency changes in a PR are reflected. | |
| - name: Install rclone | |
| run: | | |
| conda activate datashuttle-test | |
| conda install -c conda-forge rclone | |
| - name: Install datashuttle from repo | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install .[dev] | |
| - name: Install pass on Linux | |
| # this is required for Rclone config encryption | |
| if: runner.os == 'Linux' | |
| run: | | |
| set -euo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y pass gnupg git | |
| # Create a dedicated GPG home for this job | |
| export GNUPGHOME="$(mktemp -d)" | |
| echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV" # <-- make it available to later steps | |
| # Generate a non-interactive key (no passphrase), no expiry | |
| gpg --batch --yes --pinentry-mode loopback --passphrase '' \ | |
| --quick-gen-key "CI Key <[email protected]>" default default 0 | |
| # Initialize pass with the key fingerprint (more robust than UID) | |
| FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')" | |
| pass init "$FPR" | |
| # run SSH tests only on Linux because Windows and macOS | |
| # are already run within a virtual container and so cannot | |
| # run Linux containers because nested containerisation is disabled. | |
| - name: Test SSH (Linux only) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo service mysql stop # free up port 3306 for ssh tests | |
| pytest tests/tests_transfers/ssh | |
| - name: Test Google Drive | |
| env: | |
| GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} | |
| GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} | |
| GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }} | |
| GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }} | |
| run: | | |
| pytest tests/tests_transfers/gdrive | |
| - name: Test AWS | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} | |
| run: | | |
| pytest tests/tests_transfers/aws | |
| - name: All Other Tests | |
| run: | | |
| pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws | |
| build_sdist_wheels: | |
| name: Build source distribution | |
| needs: [test] | |
| if: github.event_name == 'push' && github.ref_type == 'tag' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: neuroinformatics-unit/actions/build_sdist_wheels@v2 | |
| upload_all: | |
| name: Publish build distributions | |
| needs: [build_sdist_wheels] | |
| if: github.event_name == 'push' && github.ref_type == 'tag' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v5 | |
| with: | |
| name: artifact | |
| path: dist | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.TWINE_API_KEY }} |