fix(integration_tests): permissions and task queue cleanup

Author: cabljac

integration_test/README.md

@@ -248,8 +248,45 @@ Add test file mapping in the case statement (lines 175-199).
 
 ## Authentication
 
+### Local Development
 Place your service account key at `sa.json` in the root directory. This file is git-ignored.
 
+### Cloud Build
+Cloud Build uses Application Default Credentials (ADC) automatically. However, the Cloud Build service account requires specific permissions for the Google Cloud services used in tests:
+
+**Required IAM Roles for Cloud Build Service Account:**
+- `roles/cloudtasks.admin` - For Cloud Tasks integration tests
+- `roles/cloudscheduler.admin` - For Cloud Scheduler integration tests  
+- `roles/cloudtestservice.testAdmin` - For Firebase Test Lab integration tests
+- `roles/firebase.admin` - For Firebase services (already included)
+- `roles/pubsub.publisher` - For Pub/Sub integration tests (already included)
+
+**Multi-Project Setup:**
+Tests deploy to multiple projects (typically one for V1 tests and one for V2 tests). The Cloud Build service account needs the above permissions on **all target projects**:
+
+```bash
+# Grant permissions to each target project
+gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+  --member="serviceAccount:[email protected]" \
+  --role="roles/cloudtasks.admin"
+
+gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+  --member="serviceAccount:[email protected]" \
+  --role="roles/cloudscheduler.admin"
+
+gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+  --member="serviceAccount:[email protected]" \
+  --role="roles/cloudtestservice.testAdmin"
+
+gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+  --member="serviceAccount:[email protected]" \
+  --role="roles/firebase.admin"
+```
+
+Replace:
+- `TARGET_PROJECT_ID` with each project where tests will be deployed
+- `CLOUD_BUILD_PROJECT_NUMBER` with the project number where Cloud Build runs
+
 ## Test Isolation
 
 Each test run gets a unique TEST_RUN_ID that:
@@ -284,10 +321,37 @@ Format: `t_<timestamp>_<random>` (e.g., `t_1757979490_xkyqun`)
 - Ensure TEST_RUN_ID environment variable is set
 - Check test logs in logs/ directory
 
+### Permission Errors in Cloud Build
+If you see authentication errors like "Could not refresh access token" or "Permission denied":
+- Verify Cloud Build service account has required IAM roles on all target projects
+- Check project numbers: `gcloud projects describe PROJECT_ID --format="value(projectNumber)"`
+- Grant missing permissions to each target project:
+  ```bash
+  # For Cloud Tasks
+  gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+    --member="serviceAccount:[email protected]" \
+    --role="roles/cloudtasks.admin"
+  
+  # For Cloud Scheduler  
+  gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+    --member="serviceAccount:[email protected]" \
+    --role="roles/cloudscheduler.admin"
+  
+  # For Test Lab
+  gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+    --member="serviceAccount:[email protected]" \
+    --role="roles/cloudtestservice.testAdmin"
+  
+  # For Firebase services
+  gcloud projects add-iam-policy-binding TARGET_PROJECT_ID \
+    --member="serviceAccount:[email protected]" \
+    --role="roles/firebase.admin"
+  ```
+
 ### Cleanup Issues
 - Use `npm run cleanup:list` to find orphaned test runs
 - Manual cleanup: `firebase functions:delete <function-name> --project <project-id> --force`
-- Check for leftover test functions: `firebase functions:list --project functions-integration-tests | grep Test`
+- Check for leftover test functions: `firebase functions:list --project PROJECT_ID | grep Test`
 - Check Firestore/Database console for orphaned test data
 
 ## Benefits

integration_test/scripts/run-tests.js

@@ -621,6 +621,51 @@ class TestRunner {
         // Ignore cleanup errors
       }
     }
+
+    // Clean up Cloud Tasks queues if tasks tests were run
+    if (metadata.suites.some((s) => s.name.includes("tasks"))) {
+      await this.cleanupCloudTasksQueues(metadata);
+    }
+  }
+
+  /**
+   * Clean up Cloud Tasks queues created by tests
+   */
+  async cleanupCloudTasksQueues(metadata) {
+    this.log("   Cleaning up Cloud Tasks queues...", "warn");
+
+    const region = metadata.region || DEFAULT_REGION;
+    const projectId = metadata.projectId;
+
+    // Extract queue names from metadata (function names become queue names in v1)
+    const queueNames = new Set();
+    for (const suite of metadata.suites || []) {
+      if (suite.name.includes("tasks")) {
+        for (const func of suite.functions || []) {
+          if (func.name && func.name.includes("Tests")) {
+            // Function name becomes the queue name in v1
+            queueNames.add(func.name);
+          }
+        }
+      }
+    }
+
+    // Delete each queue
+    for (const queueName of queueNames) {
+      try {
+        this.log(`   Deleting Cloud Tasks queue: ${queueName}`, "warn");
+
+        // Try gcloud command to delete the queue
+        await this.exec(
+          `gcloud tasks queues delete ${queueName} --location=${region} --project=${projectId} --quiet`,
+          { silent: true }
+        );
+        this.log(`   ✅ Deleted Cloud Tasks queue: ${queueName}`);
+      } catch (error) {
+        // Queue might not exist or already deleted, ignore errors
+        this.log(`   ⚠️  Could not delete queue ${queueName}: ${error.message}`, "warn");
+      }
+    }
   }
 
   /**
@@ -719,13 +764,76 @@ class TestRunner {
       }
     }
 
+    // Clean up orphaned Cloud Tasks queues
+    await this.cleanupOrphanedCloudTasksQueues();
+
     // Clean up generated directory
     if (existsSync(GENERATED_DIR)) {
       this.log("   Cleaning up generated directory...", "warn");
       rmSync(GENERATED_DIR, { recursive: true, force: true });
     }
   }
 
+  /**
+   * Clean up orphaned Cloud Tasks queues from previous test runs
+   */
+  async cleanupOrphanedCloudTasksQueues() {
+    this.log("   Checking for orphaned Cloud Tasks queues...", "warn");
+
+    const projects = ["functions-integration-tests", "functions-integration-tests-v2"];
+    const region = DEFAULT_REGION;
+
+    for (const projectId of projects) {
+      this.log(`   Checking Cloud Tasks queues in project: ${projectId}`, "warn");
+
+      try {
+        // List all queues in the project
+        const result = await this.exec(
+          `gcloud tasks queues list --location=${region} --project=${projectId} --format="value(name)"`,
+          { silent: true }
+        );
+
+        const queueNames = result.stdout
+          .split("\n")
+          .map((line) => line.trim())
+          .filter((line) => line.length > 0);
+
+        // Find test queues (containing "Tests" and test run ID pattern)
+        const testQueues = queueNames.filter((queueName) => {
+          const queueId = queueName.split("/").pop(); // Extract queue ID from full path
+          return queueId && queueId.match(/Tests.*t[a-z0-9]{7,10}/);
+        });
+
+        if (testQueues.length > 0) {
+          this.log(
+            `   Found ${testQueues.length} orphaned test queue(s) in ${projectId}. Cleaning up...`,
+            "warn"
+          );
+
+          for (const queuePath of testQueues) {
+            try {
+              const queueId = queuePath.split("/").pop();
+              this.log(`   Deleting orphaned queue: ${queueId}`, "warn");
+
+              await this.exec(
+                `gcloud tasks queues delete ${queueId} --location=${region} --project=${projectId} --quiet`,
+                { silent: true }
+              );
+              this.log(`   ✅ Deleted orphaned queue: ${queueId}`);
+            } catch (error) {
+              this.log(`   ⚠️  Could not delete queue ${queuePath}: ${error.message}`, "warn");
+            }
+          }
+        } else {
+          this.log(`   ✅ No orphaned test queues found in ${projectId}`, "success");
+        }
+      } catch (e) {
+        // Project might not be accessible or Cloud Tasks API not enabled
+        this.log(`   ⚠️  Could not check queues in ${projectId}: ${e.message}`, "warn");
+      }
+    }
+  }
+
   /**
    * Run a single suite
    */