diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml new file mode 100644 index 00000000..0a88e649 --- /dev/null +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -0,0 +1,188 @@ +substitutions: + _REGION: "us-central1" + _ZONE: "us-central1-a" + _SHORT_BUILD_ID: ${BUILD_ID:0:8} + +steps: + # Step 1: Create a unique standard GCS bucket for the test run. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-standard-bucket" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "create" + - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--location=${_REGION}" + waitFor: ["-"] + + # Step 2: Create a unique versioned GCS bucket for the test run. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-versioned-bucket" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "create" + - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--location=${_REGION}" + waitFor: ["-"] + + # Step 2a: Enable versioning on the versioned bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "enable-bucket-versioning" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "update" + - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" + - "--versioning" + waitFor: + - "create-versioned-bucket" + + # Step 3: Create a GCE VM to run the tests. + # The VM is created in the same zone as the buckets to test rapid storage features. + # It's given the 'cloud-platform' scope to allow it to access GCS and other services. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "create" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--zone=${_ZONE}" + - "--machine-type=e2-medium" + - "--image-family=debian-13" + - "--image-project=debian-cloud" + - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" + - "--scopes=https://www.googleapis.com/auth/cloud-platform" # Full access to project APIs + - "--metadata=enable-oslogin=TRUE" + waitFor: ["-"] + + # Step 4: Run the integration tests inside the newly created VM. + # This step uses 'gcloud compute ssh' to execute a remote script. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-tests-on-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + # Wait for the VM to be fully initialized and SSH to be ready. + for i in {1..10}; do + if gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="echo VM is ready"; then + break + fi + echo "Waiting for VM to become available... (attempt $$i/10)" + sleep 15 + done + + # Script to be executed on the VM. + # This script installs dependencies, sets environment variables, and runs pytest. + VM_SCRIPT=" + set -e + echo '--- Installing git and cloning repository on VM ---' + sudo apt-get update > /dev/null + sudo apt-get install -y git python3-pip python3-venv fuse fuse3 libfuse2 > /dev/null + + # Clone the repository and checkout the specific commit from the build trigger. + git clone https://github.com/${_REPO_OWNER}/gcsfs.git + cd gcsfs + git checkout ${COMMIT_SHA} + + + echo '--- Installing Python and dependencies on VM ---' + python3 -m venv env + source env/bin/activate + + pip install --upgrade pip > /dev/null + # Install testing libraries explicitly, as they are not in setup.py + pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null + pip install -e . > /dev/null + + echo '--- Preparing test environment on VM ---' + export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' + export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' + + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com + export GCSFS_TEST_PROJECT=${PROJECT_ID} + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} + + echo '--- Running standard tests on VM ---' + pytest -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' \ + --log-date-format='%H:%M:%S' \ + gcsfs/ \ + --deselect gcsfs/tests/test_core.py::test_sign + + echo '--- Running Zonal tests on VM ---' + export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="true" + pytest -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' \ + --log-date-format='%H:%M:%S' \ + gcsfs/tests/test_extended_gcsfs.py + " + + # Execute the script on the VM via SSH. + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="$$VM_SCRIPT" + waitFor: + - "create-vm" + - "create-standard-bucket" + - "enable-bucket-versioning" + + # --- Cleanup Steps --- + + # Step 5: Delete the GCE VM. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "delete" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--zone=${_ZONE}" + - "--quiet" + waitFor: + - "run-tests-on-vm" + + # Step 6: Delete the standard GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-standard-bucket" + entrypoint: "gcloud" + args: + [ + "storage", + "rm", + "--recursive", + "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}", + ] + waitFor: + - "run-tests-on-vm" + + # Step 7: Delete the versioned GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-versioned-bucket" + entrypoint: "gcloud" + args: + [ + "storage", + "rm", + "--recursive", + "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}", + ] + waitFor: + - "run-tests-on-vm" + +timeout: "3600s" # 60 minutes + +options: + logging: CLOUD_LOGGING_ONLY + pool: + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"