From 05eca6ad4cefb752e3f3d95f10c2421d602bc123 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Sun, 30 Nov 2025 18:14:30 +0000 Subject: [PATCH 1/4] add cloudbuild ci pipeline configuration --- cloudbuild/e2e-tests-cloudbuild.yaml | 220 +++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 cloudbuild/e2e-tests-cloudbuild.yaml diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml new file mode 100644 index 00000000..5a0b7e1e --- /dev/null +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -0,0 +1,220 @@ +substitutions: + _REGION: "us-central1" + _ZONE: "us-central1-a" + _SHORT_BUILD_ID: ${BUILD_ID:0:8} + +steps: + # Step 1: Create a unique standard GCS bucket for the test run. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-standard-bucket" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "create" + - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--location=${_REGION}" + waitFor: ["-"] + + # Step 2: Create a unique versioned GCS bucket for the test run. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-versioned-bucket" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "create" + - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--location=${_REGION}" + waitFor: ["-"] + + # Step 2a: Enable versioning on the versioned bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "enable-bucket-versioning" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "update" + - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" + - "--versioning" + waitFor: + - "create-versioned-bucket" + + # Step 3: Create a unique Zonal GCS bucket for the test run. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-zonal-bucket" + entrypoint: "gcloud" + args: + - "storage" + - "buckets" + - "create" + - "gs://gcsfs-test-zonal-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--location=${_REGION}" + - "--placement=${_ZONE}" + - "--default-storage-class=RAPID" + - "--enable-hierarchical-namespace" + - "--uniform-bucket-level-access" + waitFor: ["-"] + + # Step 4: Create a GCE VM to run the tests. + # The VM is created in the same zone as the buckets to test rapid storage features. + # It's given the 'cloud-platform' scope to allow it to access GCS and other services. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "create" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--project=${PROJECT_ID}" + - "--zone=${_ZONE}" + - "--machine-type=e2-medium" + - "--image-family=debian-13" + - "--image-project=debian-cloud" + - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" + - "--scopes=https://www.googleapis.com/auth/cloud-platform" # Full access to project APIs + - "--metadata=enable-oslogin=TRUE" + waitFor: ["-"] + + # Step 5: Run the integration tests inside the newly created VM. + # This step uses 'gcloud compute ssh' to execute a remote script. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-tests-on-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + # Wait for the VM to be fully initialized and SSH to be ready. + for i in {1..10}; do + if gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="echo VM is ready"; then + break + fi + echo "Waiting for VM to become available... (attempt $$i/10)" + sleep 15 + done + + # Script to be executed on the VM. + # This script installs dependencies, sets environment variables, and runs pytest. + VM_SCRIPT=" + set -e + echo '--- Installing git and cloning repository on VM ---' + sudo apt-get update + sudo apt-get install -y git python3-pip python3-venv fuse fuse3 libfuse2 + + # Clone the repository and checkout the specific commit from the build trigger. + git clone https://github.com/ankitaluthra1/gcsfs.git + cd gcsfs + git checkout ${COMMIT_SHA} + + + echo '--- Installing Python and dependencies on VM ---' + python3 -m venv env + source env/bin/activate + + pip install --upgrade pip + # Install testing libraries explicitly, as they are not in setup.py + pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage + pip install -e . + + echo '--- Preparing test environment on VM ---' + export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' + export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' + export GCSFS_ZONAL_TEST_BUCKET='gcsfs-test-zonal-${_SHORT_BUILD_ID}' + + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com + export GCSFS_TEST_PROJECT=${PROJECT_ID} + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} + + echo '--- Running standard tests on VM ---' + pytest -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' \ + --log-date-format='%H:%M:%S' \ + gcsfs/ \ + --deselect gcsfs/tests/test_core.py::test_sign + + echo '--- Running Zonal tests on VM ---' + export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="true" + pytest -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' \ + --log-date-format='%H:%M:%S' \ + gcsfs/tests/test_extended_gcsfs.py + " + + # Execute the script on the VM via SSH. + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="$$VM_SCRIPT" + waitFor: + - "create-vm" + - "create-standard-bucket" + - "create-zonal-bucket" + - "enable-bucket-versioning" + + # --- Cleanup Steps --- + + # Step 6: Delete the GCE VM. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "delete" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--zone=${_ZONE}" + - "--quiet" + waitFor: + - "run-tests-on-vm" + + # Step 7: Delete the standard GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-standard-bucket" + entrypoint: "gcloud" + args: + [ + "storage", + "rm", + "--recursive", + "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}", + ] + waitFor: + - "run-tests-on-vm" + + # Step 8: Delete the versioned GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-versioned-bucket" + entrypoint: "gcloud" + args: + [ + "storage", + "rm", + "--recursive", + "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}", + ] + waitFor: + - "run-tests-on-vm" + + # Step 9: Delete the zonal GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-zonal-bucket" + entrypoint: "gcloud" + args: + [ + "storage", + "rm", + "--recursive", + "gs://gcsfs-test-zonal-${_SHORT_BUILD_ID}", + ] + waitFor: + - "run-tests-on-vm" + +timeout: "3600s" # 60 minutes + +options: + logging: CLOUD_LOGGING_ONLY + pool: + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" From e04ff7ff0197539c240ec58d55fef17f04dc1786 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Sun, 30 Nov 2025 19:45:27 +0000 Subject: [PATCH 2/4] remove zonal bucket creation We need a pre setup zonal bucket as we don't have the writes implemented yet for zonal buckets. A bucket will need to be passed through substitution variable containing the required data for tests to run --- cloudbuild/e2e-tests-cloudbuild.yaml | 44 ++++------------------------ 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 5a0b7e1e..6f709a7f 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -43,24 +43,7 @@ steps: waitFor: - "create-versioned-bucket" - # Step 3: Create a unique Zonal GCS bucket for the test run. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-zonal-bucket" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-zonal-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - - "--placement=${_ZONE}" - - "--default-storage-class=RAPID" - - "--enable-hierarchical-namespace" - - "--uniform-bucket-level-access" - waitFor: ["-"] - - # Step 4: Create a GCE VM to run the tests. + # Step 3: Create a GCE VM to run the tests. # The VM is created in the same zone as the buckets to test rapid storage features. # It's given the 'cloud-platform' scope to allow it to access GCS and other services. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" @@ -81,7 +64,7 @@ steps: - "--metadata=enable-oslogin=TRUE" waitFor: ["-"] - # Step 5: Run the integration tests inside the newly created VM. + # Step 4: Run the integration tests inside the newly created VM. # This step uses 'gcloud compute ssh' to execute a remote script. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "run-tests-on-vm" @@ -125,7 +108,7 @@ steps: echo '--- Preparing test environment on VM ---' export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' - export GCSFS_ZONAL_TEST_BUCKET='gcsfs-test-zonal-${_SHORT_BUILD_ID}' + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' export STORAGE_EMULATOR_HOST=https://storage.googleapis.com export GCSFS_TEST_PROJECT=${PROJECT_ID} @@ -151,12 +134,11 @@ steps: waitFor: - "create-vm" - "create-standard-bucket" - - "create-zonal-bucket" - "enable-bucket-versioning" # --- Cleanup Steps --- - # Step 6: Delete the GCE VM. + # Step 5: Delete the GCE VM. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-vm" entrypoint: "gcloud" @@ -170,7 +152,7 @@ steps: waitFor: - "run-tests-on-vm" - # Step 7: Delete the standard GCS bucket. + # Step 6: Delete the standard GCS bucket. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-standard-bucket" entrypoint: "gcloud" @@ -184,7 +166,7 @@ steps: waitFor: - "run-tests-on-vm" - # Step 8: Delete the versioned GCS bucket. + # Step 7: Delete the versioned GCS bucket. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-versioned-bucket" entrypoint: "gcloud" @@ -198,20 +180,6 @@ steps: waitFor: - "run-tests-on-vm" - # Step 9: Delete the zonal GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-zonal-bucket" - entrypoint: "gcloud" - args: - [ - "storage", - "rm", - "--recursive", - "gs://gcsfs-test-zonal-${_SHORT_BUILD_ID}", - ] - waitFor: - - "run-tests-on-vm" - timeout: "3600s" # 60 minutes options: From 0d2893b382aa067c6b6624656feb6f6a763f77c2 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Sun, 30 Nov 2025 20:02:58 +0000 Subject: [PATCH 3/4] omit the standard environment setup logs Only print the errors and redirect standard logs to /dev/null --- cloudbuild/e2e-tests-cloudbuild.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 6f709a7f..2460c824 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -87,8 +87,8 @@ steps: VM_SCRIPT=" set -e echo '--- Installing git and cloning repository on VM ---' - sudo apt-get update - sudo apt-get install -y git python3-pip python3-venv fuse fuse3 libfuse2 + sudo apt-get update > /dev/null + sudo apt-get install -y git python3-pip python3-venv fuse fuse3 libfuse2 > /dev/null # Clone the repository and checkout the specific commit from the build trigger. git clone https://github.com/ankitaluthra1/gcsfs.git @@ -100,10 +100,10 @@ steps: python3 -m venv env source env/bin/activate - pip install --upgrade pip + pip install --upgrade pip > /dev/null # Install testing libraries explicitly, as they are not in setup.py - pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage - pip install -e . + pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null + pip install -e . > /dev/null echo '--- Preparing test environment on VM ---' export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' From f01cbadb527f4e3df4dc75a538d5ffdc22be9865 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Mon, 1 Dec 2025 07:59:36 +0000 Subject: [PATCH 4/4] Use repo_owner substitution rule Instead of hardcoding the github repository to be cloned, use substitution rule so same cloudbuild file can be used for public repo --- cloudbuild/e2e-tests-cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 2460c824..0a88e649 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -91,7 +91,7 @@ steps: sudo apt-get install -y git python3-pip python3-venv fuse fuse3 libfuse2 > /dev/null # Clone the repository and checkout the specific commit from the build trigger. - git clone https://github.com/ankitaluthra1/gcsfs.git + git clone https://github.com/${_REPO_OWNER}/gcsfs.git cd gcsfs git checkout ${COMMIT_SHA}