Merge pull request github#5041 from github/parallelize-collection-tests

Parallelize collection tests into 4 shards

Author: ahpook

.github/workflows/test.yml

@@ -16,10 +16,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test_type:
-          - collections
-          - topics
-          - all
+        include:
+          - test_type: topics
+          - test_type: collections
+            shard: 0
+            total_shards: 4
+          - test_type: collections
+            shard: 1
+            total_shards: 4
+          - test_type: collections
+            shard: 2
+            total_shards: 4
+          - test_type: collections
+            shard: 3
+            total_shards: 4
+          - test_type: all
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6.0.1
@@ -58,3 +69,5 @@ jobs:
           TOPIC_FILES: ${{ steps.topics.outputs.changed }}
           COLLECTION_FILES: ${{ steps.collections.outputs.changed }}
           TEST_ALL_FILES: ${{ steps.all.outputs.changed }}
+          COLLECTION_SHARD: ${{ matrix.shard }}
+          COLLECTION_TOTAL_SHARDS: ${{ matrix.total_shards }}

test/collections_test_helper.rb

@@ -53,7 +53,19 @@ def dirs_to_test
 end
 
 def collections
-  collection_dirs.map { |dir_path| File.basename(dir_path) }
+  all = collection_dirs.map { |dir_path| File.basename(dir_path) }
+  shard_collections(all)
+end
+
+def shard_collections(all_collections)
+  shard = ENV["COLLECTION_SHARD"]&.to_i
+  total_shards = ENV["COLLECTION_TOTAL_SHARDS"]&.to_i
+
+  return all_collections unless !shard.nil? && !total_shards.nil? && total_shards > 1
+
+  # Sort alphabetically for deterministic sharding
+  sorted = all_collections.sort
+  sorted.select.with_index { |_, i| i % total_shards == shard }
 end
 
 def items_for_collection(collection)