Test Object Storage Adapter #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Object Storage Adapter | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| INT_TEST_JAVA_RUNTIME_VERSION: | |
| description: JDK version used to run the integration test | |
| type: choice | |
| required: false | |
| default: '8' | |
| options: | |
| - '8' | |
| - '11' | |
| - '17' | |
| - '21' | |
| INT_TEST_JAVA_RUNTIME_VENDOR: | |
| description: Vendor of the JDK used to run the integration test | |
| type: choice | |
| required: false | |
| default: 'temurin' | |
| options: | |
| - 'corretto' | |
| - 'microsoft' | |
| - 'oracle' | |
| - 'temurin' | |
| env: | |
| TERM: dumb | |
| JAVA_VERSION: '8' | |
| JAVA_VENDOR: 'temurin' | |
| INT_TEST_JAVA_RUNTIME_VERSION: "${{ github.event_name != 'workflow_dispatch' && '8' || inputs.INT_TEST_JAVA_RUNTIME_VERSION }}" | |
| INT_TEST_JAVA_RUNTIME_VENDOR: "${{ github.event_name != 'workflow_dispatch' && 'temurin' || inputs.INT_TEST_JAVA_RUNTIME_VENDOR }}" | |
| # Gradle will parse 'ORG_GRADLE_PROJECT_<project_property_name>' environment variables as project properties. | |
| # The following variables configure the 'com.scalar.db.jdk-configuration' Gradle plugin. | |
| ORG_GRADLE_PROJECT_javaVersion: '8' | |
| ORG_GRADLE_PROJECT_javaVendor: 'temurin' | |
| ORG_GRADLE_PROJECT_integrationTestJavaRuntimeVersion: "${{ github.event_name != 'workflow_dispatch' && '8' || inputs.INT_TEST_JAVA_RUNTIME_VERSION }}" | |
| ORG_GRADLE_PROJECT_integrationTestJavaRuntimeVendor: "${{ github.event_name != 'workflow_dispatch' && 'temurin' || inputs.INT_TEST_JAVA_RUNTIME_VENDOR }}" | |
| # This variable evaluates to: if {!(Temurin JDK 8) && !(Oracle JDK)} then {true} else {false} | |
| # Oracle JDK that are linux compatible and publicly available through direct download exist for all LTS versions | |
| SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK: "${{ (github.event_name == 'workflow_dispatch' && !(inputs.INT_TEST_JAVA_RUNTIME_VERSION == '8' && inputs.INT_TEST_JAVA_RUNTIME_VENDOR == 'temurin') && !(inputs.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle')) && 'true' || 'false' }}" | |
| INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"' | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} | |
| S3_REGION: us-east-1 | |
| S3_BUCKET_BASE_NAME: s3-scalardb-test-bucket | |
| CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }} | |
| CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} | |
| CLOUD_STORAGE_BUCKET_BASE_NAME: scalardb-test-bucket | |
| jobs: | |
| integration-test-s3: | |
| name: S3 integration test (${{ matrix.test_group.label }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_group: | |
| - label: consensus_commit_default | |
| tests_filter: '--tests "**.ConsensusCommit**"' | |
| bucket_suffix: consensus-commit | |
| group_commit_enabled: false | |
| - label: consensus_commit_with_group_commit | |
| tests_filter: '--tests "**.ConsensusCommit**"' | |
| bucket_suffix: consensus-commit-gc | |
| group_commit_enabled: true | |
| - label: storage_scan | |
| tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"' | |
| bucket_suffix: storage-scan | |
| group_commit_enabled: false | |
| - label: storage_wrapper | |
| tests_filter: '--tests "**.ObjectStorageWrapper**"' | |
| bucket_suffix: storage-wrapper | |
| group_commit_enabled: false | |
| - label: storage_admin | |
| tests_filter: '--tests "**.ObjectStorageAdmin**"' | |
| bucket_suffix: storage-admin | |
| group_commit_enabled: false | |
| - label: storage_cm | |
| tests_filter: '--tests "**.ObjectStorageConditionalMutation**"' | |
| bucket_suffix: storage-cm | |
| group_commit_enabled: false | |
| - label: storage_others | |
| tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"' | |
| bucket_suffix: storage | |
| group_commit_enabled: false | |
| - label: two_phase_consensus_commit | |
| tests_filter: '--tests "**.TwoPhaseConsensusCommit**"' | |
| bucket_suffix: 2pcc | |
| group_commit_enabled: false | |
| - label: single_crud_operation_transaction | |
| tests_filter: '--tests "**.SingleCrudOperationTransaction**"' | |
| bucket_suffix: single-crud | |
| group_commit_enabled: false | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }}) | |
| uses: actions/setup-java@v5 | |
| with: | |
| java-version: ${{ env.JAVA_VERSION }} | |
| distribution: ${{ env.JAVA_VENDOR }} | |
| - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test | |
| uses: actions/setup-java@v5 | |
| if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}} | |
| with: | |
| java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} | |
| distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }} | |
| - name: Login to Oracle container registry | |
| uses: docker/login-action@v3 | |
| if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }} | |
| with: | |
| registry: container-registry.oracle.com | |
| username: ${{ secrets.OCR_USERNAME }} | |
| password: ${{ secrets.OCR_TOKEN }} | |
| - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test | |
| if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }} | |
| run: | | |
| container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}") | |
| docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id" | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v5 | |
| - name: Execute Gradle 'integrationTestObjectStorage' task | |
| run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.AWS_ACCESS_KEY_ID }}' -Dscalardb.object_storage.password='${{ env.AWS_SECRET_ACCESS_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }} | |
| - name: Upload Gradle test reports | |
| if: always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: s3_integration_test_reports_${{ matrix.test_group.label }} | |
| path: core/build/reports/tests/integrationTestObjectStorage | |
| integration-test-cloud-storage: | |
| name: Cloud Storage integration test (${{ matrix.test_group.label }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_group: | |
| - label: consensus_commit_default | |
| tests_filter: '--tests "**.ConsensusCommit**"' | |
| bucket_suffix: consensus-commit | |
| group_commit_enabled: false | |
| - label: consensus_commit_with_group_commit | |
| tests_filter: '--tests "**.ConsensusCommit**"' | |
| bucket_suffix: consensus-commit-gc | |
| group_commit_enabled: true | |
| - label: storage_scan | |
| tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"' | |
| bucket_suffix: storage-scan | |
| group_commit_enabled: false | |
| - label: storage_wrapper | |
| tests_filter: '--tests "**.ObjectStorageWrapper**"' | |
| bucket_suffix: storage-wrapper | |
| group_commit_enabled: false | |
| - label: storage_admin | |
| tests_filter: '--tests "**.ObjectStorageAdmin**"' | |
| bucket_suffix: storage-admin | |
| group_commit_enabled: false | |
| - label: storage_cm | |
| tests_filter: '--tests "**.ObjectStorageConditionalMutation**"' | |
| bucket_suffix: storage-cm | |
| group_commit_enabled: false | |
| - label: storage_others | |
| tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"' | |
| bucket_suffix: storage | |
| group_commit_enabled: false | |
| - label: two_phase_consensus_commit | |
| tests_filter: '--tests "**.TwoPhaseConsensusCommit**"' | |
| bucket_suffix: 2pcc | |
| group_commit_enabled: false | |
| - label: single_crud_operation_transaction | |
| tests_filter: '--tests "**.SingleCrudOperationTransaction**"' | |
| bucket_suffix: single-crud | |
| group_commit_enabled: false | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }}) | |
| uses: actions/setup-java@v5 | |
| with: | |
| java-version: ${{ env.JAVA_VERSION }} | |
| distribution: ${{ env.JAVA_VENDOR }} | |
| - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test | |
| uses: actions/setup-java@v5 | |
| if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}} | |
| with: | |
| java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} | |
| distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }} | |
| - name: Login to Oracle container registry | |
| uses: docker/login-action@v3 | |
| if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }} | |
| with: | |
| registry: container-registry.oracle.com | |
| username: ${{ secrets.OCR_USERNAME }} | |
| password: ${{ secrets.OCR_TOKEN }} | |
| - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test | |
| if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }} | |
| run: | | |
| container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}") | |
| docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id" | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v5 | |
| - name: Execute Gradle 'integrationTestObjectStorage' task | |
| run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=${{ env.CLOUD_STORAGE_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.CLOUD_STORAGE_PROJECT_ID }}' -Dscalardb.object_storage.password='${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }} | |
| - name: Upload Gradle test reports | |
| if: always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: cloud_storage_integration_test_reports_${{ matrix.test_group.label }} | |
| path: core/build/reports/tests/integrationTestObjectStorage |