Add ci pipeline using cloudbuild to run tests against actual GCS

cloudbuild/e2e-tests-cloudbuild.yaml

@@ -0,0 +1,188 @@
+substitutions:
+  _REGION: "us-central1"
+  _ZONE: "us-central1-a"
+  _SHORT_BUILD_ID: ${BUILD_ID:0:8}
+
+steps:
+  # Step 1: Create a unique standard GCS bucket for the test run.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "create-standard-bucket"
+    entrypoint: "gcloud"
+    args:
+      - "storage"
+      - "buckets"
+      - "create"
+      - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}"
+      - "--project=${PROJECT_ID}"
+      - "--location=${_REGION}"
+    waitFor: ["-"]
+
+  # Step 2: Create a unique versioned GCS bucket for the test run.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "create-versioned-bucket"
+    entrypoint: "gcloud"
+    args:
+      - "storage"
+      - "buckets"
+      - "create"
+      - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}"
+      - "--project=${PROJECT_ID}"
+      - "--location=${_REGION}"
+    waitFor: ["-"]
+
+  # Step 2a: Enable versioning on the versioned bucket.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "enable-bucket-versioning"
+    entrypoint: "gcloud"
+    args:
+      - "storage"
+      - "buckets"
+      - "update"
+      - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}"
+      - "--versioning"
+    waitFor:
+      - "create-versioned-bucket"
+
+  # Step 3: Create a GCE VM to run the tests.
+  # The VM is created in the same zone as the buckets to test rapid storage features.
+  # It's given the 'cloud-platform' scope to allow it to access GCS and other services.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "create-vm"
+    entrypoint: "gcloud"
+    args:
+      - "compute"
+      - "instances"
+      - "create"
+      - "gcsfs-test-vm-${_SHORT_BUILD_ID}"
+      - "--project=${PROJECT_ID}"
+      - "--zone=${_ZONE}"
+      - "--machine-type=e2-medium"
+      - "--image-family=debian-13"
+      - "--image-project=debian-cloud"
+      - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}"
+      - "--scopes=https://www.googleapis.com/auth/cloud-platform" # Full access to project APIs
+      - "--metadata=enable-oslogin=TRUE"
+    waitFor: ["-"]
+
+  # Step 4: Run the integration tests inside the newly created VM.
+  # This step uses 'gcloud compute ssh' to execute a remote script.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "run-tests-on-vm"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        set -e
+        # Wait for the VM to be fully initialized and SSH to be ready.
+        for i in {1..10}; do
+          if gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="echo VM is ready"; then
+            break
+          fi
+          echo "Waiting for VM to become available... (attempt $$i/10)"
+          sleep 15
+        done
+
+        # Script to be executed on the VM.
+        # This script installs dependencies, sets environment variables, and runs pytest.
+        VM_SCRIPT="
+          set -e
+          echo '--- Installing git and cloning repository on VM ---'
+          sudo apt-get update > /dev/null
+          sudo apt-get install -y git python3-pip python3-venv fuse fuse3 libfuse2 > /dev/null
+
+          # Clone the repository and checkout the specific commit from the build trigger.
+          git clone https://github.com/${_REPO_OWNER}/gcsfs.git
+          cd gcsfs
+          git checkout ${COMMIT_SHA}
+
+
+          echo '--- Installing Python and dependencies on VM ---'
+          python3 -m venv env
+          source env/bin/activate
+
+          pip install --upgrade pip > /dev/null
+          # Install testing libraries explicitly, as they are not in setup.py
+          pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null
+          pip install -e . > /dev/null
+
+          echo '--- Preparing test environment on VM ---'
+          export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}'
+          export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}'
+          export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}'
+
+          export STORAGE_EMULATOR_HOST=https://storage.googleapis.com
+          export GCSFS_TEST_PROJECT=${PROJECT_ID}
+          export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME}
+
+          echo '--- Running standard tests on VM ---'
+          pytest -vv -s \
+          --log-format='%(asctime)s %(levelname)s %(message)s' \
+          --log-date-format='%H:%M:%S' \
+          gcsfs/ \
+          --deselect gcsfs/tests/test_core.py::test_sign
+
+          echo '--- Running Zonal tests on VM ---'
+          export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="true"
+          pytest -vv -s \
+          --log-format='%(asctime)s %(levelname)s %(message)s' \
+          --log-date-format='%H:%M:%S' \
+          gcsfs/tests/test_extended_gcsfs.py
+        "
+
+        # Execute the script on the VM via SSH.
+        gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --command="$$VM_SCRIPT"
+    waitFor:
+      - "create-vm"
+      - "create-standard-bucket"
+      - "enable-bucket-versioning"
+
+  # --- Cleanup Steps ---
+
+  # Step 5: Delete the GCE VM.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "delete-vm"
+    entrypoint: "gcloud"
+    args:
+      - "compute"
+      - "instances"
+      - "delete"
+      - "gcsfs-test-vm-${_SHORT_BUILD_ID}"
+      - "--zone=${_ZONE}"
+      - "--quiet"
+    waitFor:
+      - "run-tests-on-vm"
+
+  # Step 6: Delete the standard GCS bucket.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "delete-standard-bucket"
+    entrypoint: "gcloud"
+    args:
+      [
+        "storage",
+        "rm",
+        "--recursive",
+        "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}",
+      ]
+    waitFor:
+      - "run-tests-on-vm"
+
+  # Step 7: Delete the versioned GCS bucket.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "delete-versioned-bucket"
+    entrypoint: "gcloud"
+    args:
+      [
+        "storage",
+        "rm",
+        "--recursive",
+        "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}",
+      ]
+    waitFor:
+      - "run-tests-on-vm"
+
+timeout: "3600s" # 60 minutes
+
+options:
+  logging: CLOUD_LOGGING_ONLY
+  pool:
+    name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"