Skip to content

Commit 718f218

Browse files
dbolducdbolduc
authored
ci: reshard macOS builds (#14681)
1 parent ca97e68 commit 718f218

File tree

3 files changed

+214
-49
lines changed

3 files changed

+214
-49
lines changed

‎.github/workflows/macos-bazel.yml‎

Lines changed: 79 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ jobs:
3838
- Spanner
3939
- Storage
4040
- Compute
41-
- Larger Libraries
41+
- Shard1
42+
- Shard2
4243
- Other
4344
include:
4445
- shard: Core
@@ -64,7 +65,7 @@ jobs:
6465
- shard: Compute
6566
targets:
6667
- //google/cloud/compute/...
67-
- shard: Larger Libraries
68+
- shard: Shard1
6869
targets:
6970
# Run this:
7071
#
@@ -74,24 +75,52 @@ jobs:
7475
# awk '{ s += $1; print s, $0}'
7576
#
7677
# and pick the approximate midpoint
77-
- //google/cloud/servicecontrol/...
78+
#
79+
# Also, it is a good idea to keep `sql` and `bigquerycontrol`
80+
# together, as they both require our internal REST libraries.
81+
- //google/cloud/bigquerycontrol/...
82+
- +//google/cloud/appengine/...
83+
- +//google/cloud/dataproc/...
84+
- +//google/cloud/monitoring/...
85+
- +//google/cloud/retail/...
86+
- +//google/cloud/discoveryengine/...
87+
- +//google/cloud/sql/...
88+
- +//google/cloud/dialogflow_cx/...
89+
- +//google/cloud/dialogflow_es/...
90+
- +//google/cloud/aiplatform/...
91+
- shard: Shard2
92+
targets:
93+
- //google/cloud/networkconnectivity/...
94+
- +//google/cloud/networkservices/...
95+
- +//google/cloud/policytroubleshooter/...
96+
- +//google/cloud/profiler/...
97+
- +//google/cloud/redis/...
98+
- +//google/cloud/securitycenter/...
99+
- +//google/cloud/servicedirectory/...
100+
- +//google/cloud/tpu/...
101+
- +//google/cloud/trace/...
102+
- +//google/cloud/vision/...
103+
- +//google/cloud/workflows/...
104+
- +//google/cloud/beyondcorp/...
105+
- +//google/cloud/billing/...
106+
- +//google/cloud/binaryauthorization/...
107+
- +//google/cloud/gkemulticloud/...
108+
- +//google/cloud/logging/...
109+
- +//google/cloud/notebooks/...
110+
- +//google/cloud/osconfig/...
111+
- +//google/cloud/servicecontrol/...
112+
- +//google/cloud/speech/...
78113
- +//google/cloud/support/...
79114
- +//google/cloud/video/...
80115
- +//google/cloud/datacatalog/...
81116
- +//google/cloud/iam/...
82-
- +//google/cloud/kms/...
83-
- +//google/cloud/beyondcorp/...
84-
- +//google/cloud/contentwarehouse/...
85-
- +//google/cloud/dataproc/...
117+
- +//google/cloud/run/...
86118
- +//google/cloud/talent/...
87-
- +//google/cloud/retail/...
119+
- +//google/cloud/contentwarehouse/...
120+
- +//google/cloud/dataplex/...
121+
- +//google/cloud/kms/...
88122
- +//google/cloud/bigquery/...
89-
- +//google/cloud/appengine/...
90-
- +//google/cloud/monitoring/...
91-
- +//google/cloud/sql/...
92-
- +//google/cloud/dialogflow_cx/...
93-
- +//google/cloud/aiplatform/...
94-
- +//google/cloud/dialogflow_es/...
123+
- +//google/cloud/resourcemanager/...
95124
- shard: Other
96125
targets:
97126
- //...
@@ -113,25 +142,49 @@ jobs:
113142
- -//google/cloud/storage/...
114143
# From Compute
115144
- -//google/cloud/compute/...
116-
# From Large Libraries
145+
# From Shard1
146+
- -//google/cloud/appengine/...
147+
- -//google/cloud/dataproc/...
148+
- -//google/cloud/monitoring/...
149+
- -//google/cloud/retail/...
150+
- -//google/cloud/discoveryengine/...
151+
- -//google/cloud/sql/...
152+
- -//google/cloud/dialogflow_cx/...
153+
- -//google/cloud/dialogflow_es/...
154+
- -//google/cloud/aiplatform/...
155+
# From Shard2
156+
- -//google/cloud/networkconnectivity/...
157+
- -//google/cloud/networkservices/...
158+
- -//google/cloud/policytroubleshooter/...
159+
- -//google/cloud/profiler/...
160+
- -//google/cloud/redis/...
161+
- -//google/cloud/securitycenter/...
162+
- -//google/cloud/servicedirectory/...
163+
- -//google/cloud/tpu/...
164+
- -//google/cloud/trace/...
165+
- -//google/cloud/vision/...
166+
- -//google/cloud/workflows/...
167+
- -//google/cloud/beyondcorp/...
168+
- -//google/cloud/billing/...
169+
- -//google/cloud/binaryauthorization/...
170+
- -//google/cloud/gkemulticloud/...
171+
- -//google/cloud/logging/...
172+
- -//google/cloud/notebooks/...
173+
- -//google/cloud/osconfig/...
117174
- -//google/cloud/servicecontrol/...
175+
- -//google/cloud/speech/...
118176
- -//google/cloud/support/...
119177
- -//google/cloud/video/...
120178
- -//google/cloud/datacatalog/...
121179
- -//google/cloud/iam/...
122-
- -//google/cloud/kms/...
123-
- -//google/cloud/beyondcorp/...
124-
- -//google/cloud/contentwarehouse/...
125-
- -//google/cloud/dataproc/...
180+
- -//google/cloud/run/...
126181
- -//google/cloud/talent/...
127-
- -//google/cloud/retail/...
182+
- -//google/cloud/contentwarehouse/...
183+
- -//google/cloud/dataplex/...
184+
- -//google/cloud/kms/...
128185
- -//google/cloud/bigquery/...
129-
- -//google/cloud/appengine/...
130-
- -//google/cloud/monitoring/...
131-
- -//google/cloud/sql/...
132-
- -//google/cloud/dialogflow_cx/...
133-
- -//google/cloud/aiplatform/...
134-
- -//google/cloud/dialogflow_es/...
186+
- -//google/cloud/bigquerycontrol/...
187+
- -//google/cloud/resourcemanager/...
135188
steps:
136189
- uses: actions/checkout@v4
137190
with:

‎.github/workflows/macos-cmake.yml‎

Lines changed: 133 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,19 @@ jobs:
4040
matrix:
4141
exclude-from-full-trick: [ true ]
4242
os: [ macos-14 ]
43-
shard: [ Core, Compute, Shard1, Other ]
43+
shard: [ Core1, Core2, Compute, Shard1, Shard2, Shard3, Shard4, Other ]
4444
exclude:
4545
# Only full builds include shards with generated code.
4646
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
4747
shard: Compute
4848
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
4949
shard: Shard1
50+
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
51+
shard: Shard2
52+
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
53+
shard: Shard3
54+
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
55+
shard: Shard4
5056
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
5157
shard: Other
5258
steps:
@@ -64,55 +70,164 @@ jobs:
6470
shell: bash
6571
run: |
6672
echo "vcpkg-version=$(cat ci/etc/vcpkg-version.txt)" >> "${GITHUB_OUTPUT}"
67-
core_features=(
73+
core1_features=(
6874
bigtable
75+
spanner
76+
)
77+
core2_features=(
6978
pubsub
7079
pubsublite
71-
spanner
7280
storage
7381
)
7482
# These are the libraries with the most "clients". To build the list
7583
# run something like this and find the midpoint:
7684
#
7785
# git grep -l 'class.*Client' 'google/cloud/**_client.h' |
78-
# egrep -v "(bigtable/|internal/|pubsub/|spanner/|storage/)" |
86+
# egrep -v "(bigtable/|internal/|pubsub/|pubsublite/|spanner/|storage/|testing_util/)" |
7987
# cut -f -3 -d/| sort | uniq -c | sort -n |
8088
# awk '{ s += $1; print s, $0}'
8189
#
8290
shard1_features=(
91+
bigquerycontrol
92+
dataproc
93+
monitoring
94+
retail
95+
discoveryengine
96+
sql
97+
dialogflow_cx
98+
dialogflow_es
99+
aiplatform
100+
)
101+
shard2_features=(
83102
servicecontrol
103+
speech
84104
support
85105
video
86106
datacatalog
87107
iam
88-
kms
89-
beyondcorp
90-
contentwarehouse
91-
dataproc
108+
run
92109
talent
93-
retail
110+
contentwarehouse
111+
dataplex
112+
kms
94113
bigquery
114+
resourcemanager
95115
appengine
96-
monitoring
97-
sql
98-
dialogflow_cx
99-
aiplatform
100-
dialogflow_es
101116
)
102-
if [[ "${{ matrix.shard }}" == "Core" ]]; then
103-
features="$(printf ",%s" "${core_features[@]}")"
117+
shard3_features=(
118+
channel
119+
cloudbuild
120+
cloudcontrolspartner
121+
composer
122+
containeranalysis
123+
datastore
124+
eventarc
125+
functions
126+
iap
127+
language
128+
metastore
129+
networkconnectivity
130+
networkservices
131+
policytroubleshooter
132+
profiler
133+
redis
134+
securitycenter
135+
servicedirectory
136+
tpu
137+
trace
138+
vision
139+
workflows
140+
beyondcorp
141+
billing
142+
binaryauthorization
143+
gkemulticloud
144+
logging
145+
notebooks
146+
osconfig
147+
)
148+
shard4_features=(
149+
domains
150+
edgecontainer
151+
edgenetwork
152+
essentialcontacts
153+
filestore
154+
gkebackup
155+
gkeconnect
156+
gkehub
157+
ids
158+
managedidentities
159+
managedkafka
160+
memcache
161+
migrationcenter
162+
netapp
163+
networkmanagement
164+
networksecurity
165+
optimization
166+
orgpolicy
167+
oslogin
168+
policysimulator
169+
privateca
170+
privilegedaccessmanager
171+
publicca
172+
rapidmigrationassessment
173+
recaptchaenterprise
174+
recommender
175+
resourcesettings
176+
scheduler
177+
secretmanager
178+
securesourcemanager
179+
securitycentermanagement
180+
servicehealth
181+
servicemanagement
182+
serviceusage
183+
shell
184+
storagecontrol
185+
storageinsights
186+
storagetransfer
187+
tasks
188+
telcoautomation
189+
texttospeech
190+
timeseriesinsights
191+
translate
192+
videointelligence
193+
vmmigration
194+
vmwareengine
195+
vpcaccess
196+
webrisk
197+
websecurityscanner
198+
workstations
199+
automl
200+
)
201+
if [[ "${{ matrix.shard }}" == "Core1" ]]; then
202+
features="$(printf ",%s" "${core1_features[@]}")"
203+
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
204+
elif [[ "${{ matrix.shard }}" == "Core2" ]]; then
205+
features="$(printf ",%s" "${core2_features[@]}")"
104206
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
105207
elif [[ "${{matrix.shard}}" == "Compute" ]]; then
106208
echo "features=compute" >> "${GITHUB_OUTPUT}"
107209
elif [[ "${{ matrix.shard }}" == "Shard1" ]]; then
108210
features="$(printf ",%s" "${shard1_features[@]}")"
109211
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
212+
elif [[ "${{ matrix.shard }}" == "Shard2" ]]; then
213+
features="$(printf ",%s" "${shard2_features[@]}")"
214+
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
215+
elif [[ "${{ matrix.shard }}" == "Shard3" ]]; then
216+
features="$(printf ",%s" "${shard3_features[@]}")"
217+
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
218+
elif [[ "${{ matrix.shard }}" == "Shard4" ]]; then
219+
features="$(printf ",%s" "${shard4_features[@]}")"
220+
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
110221
else
111-
skipped_features=("${core_features[@]}")
222+
skipped_features=("${core1_features[@]}")
223+
skipped_features+=("${core2_features[@]}")
112224
skipped_features+=(compute)
113225
skipped_features+=("${shard1_features[@]}")
226+
skipped_features+=("${shard2_features[@]}")
227+
skipped_features+=("${shard3_features[@]}")
228+
skipped_features+=("${shard4_features[@]}")
114229
skipped="$(printf ",-%s" "${skipped_features[@]}")"
115-
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:2}" >> "${GITHUB_OUTPUT}"
230+
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:1}" >> "${GITHUB_OUTPUT}"
116231
fi
117232
- name: Install bash >= 5.x and Ninja
118233
run: brew install bash ninja

‎.github/workflows/windows-cmake.yml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -210,10 +210,7 @@ jobs:
210210
features="$(printf ",%s" "${shard3_features[@]}")"
211211
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
212212
else
213-
# The "-" before the first element gets consumed somewhere, so pad
214-
# it out.
215-
skipped_features=(null)
216-
skipped_features+=("${core1_features[@]}")
213+
skipped_features=("${core1_features[@]}")
217214
skipped_features+=("${core2_features[@]}")
218215
skipped_features+=(compute)
219216
skipped_features+=("${shard1_features[@]}")
@@ -224,7 +221,7 @@ jobs:
224221
# Windows.
225222
skipped_features+=(asset channel)
226223
skipped="$(printf ",-%s" "${skipped_features[@]}")"
227-
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:2}" >> "${GITHUB_OUTPUT}"
224+
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:1}" >> "${GITHUB_OUTPUT}"
228225
fi
229226
- name: Download and Install CMake
230227
shell: bash

0 commit comments

Comments
 (0)