Skip to content

Commit 7e6cc7b

Browse files
authored
ci: script schedule-triggered builds (#12751)
1 parent b312cb4 commit 7e6cc7b

File tree

7 files changed

+277
-0
lines changed

7 files changed

+277
-0
lines changed

ci/cloudbuild/README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,4 +124,18 @@ $ gcloud beta builds worker-pools create \
124124
Details of the pool can be changed with the **`update`** (rather than `create`)
125125
command.
126126

127+
## Scheduled Jobs
128+
129+
A small number of builds run on a schedule as opposed to running on each PR and
130+
CI (push) event. These builds typically use some limited resource (such as
131+
production quota), or are very slow, or we created to help other teams.
132+
133+
The jobs can be saved using:
134+
135+
```shell
136+
for job in $(ci/cloudbuild/schedule.sh --ids); do \
137+
ci/cloudbuild/schedule.sh --export "${job}" >ci/cloudbuild/schedules/${job}.json
138+
done
139+
```
140+
127141
[custom-worker-pool]: https://cloud.google.com/build/docs/custom-workers/run-builds-in-custom-worker-pool

ci/cloudbuild/schedule.sh

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/bin/bash
2+
#
3+
# Copyright 2023 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
# This script manages Google Cloud Build scheduled builds. The schedules are
18+
# defined by JSON files that live in the `schedules/` directory. To create one
19+
# of these files for an existing schedule use --export. These files can be
20+
# used to create and update schedules (--import).
21+
#
22+
# Usage: schedule.sh [options]
23+
#
24+
# Options:
25+
# --list List all schedules for this project
26+
# --ids List the IDSs of all schedules this project
27+
# --export Exports an existing schedule
28+
# --import Creates or updates a schedule
29+
# -h|--help Print this help message
30+
31+
set -euo pipefail
32+
33+
source "$(dirname "$0")/../lib/init.sh"
34+
source module ci/lib/io.sh
35+
36+
function print_usage() {
37+
# Extracts the usage from the file comment starting at line 17.
38+
sed -n '17,/^$/s/^# \?//p' "${PROGRAM_PATH}"
39+
}
40+
41+
readonly CLOUD_PROJECT="cloud-cpp-testing-resources"
42+
readonly LOCATION=us-central1
43+
44+
function list_schedules() {
45+
gcloud scheduler jobs list \
46+
--project "${CLOUD_PROJECT}" \
47+
--location "${LOCATION}"
48+
}
49+
50+
function ids_schedules() {
51+
gcloud scheduler jobs list \
52+
--project "${CLOUD_PROJECT}" \
53+
--location "${LOCATION}" \
54+
--format "value(ID)"
55+
}
56+
57+
function export_schedule() {
58+
local name="$1"
59+
gcloud scheduler jobs describe "${name}" \
60+
--project "${CLOUD_PROJECT}" \
61+
--location "${LOCATION}" \
62+
--format=json
63+
}
64+
65+
function schedule_flags() {
66+
local file="$1"
67+
# jq requires a single argument, but it is too long.
68+
local args=(
69+
'"--schedule"' .schedule
70+
'"--uri"' .httpTarget.uri
71+
'"--oauth-service-account-email"' .httpTarget.oauthToken.serviceAccountEmail
72+
'"--oauth2-service-account-scope"' .httpTarget.oauthToken.scope
73+
)
74+
local script
75+
script="$(print ", %s" "${args[@]}")"
76+
jq -r "${script:2}" <"${file}"
77+
}
78+
79+
function import_schedule() {
80+
local file="$1"
81+
local id
82+
id="$(basename "${file}" .json)"
83+
local -a flags
84+
mapfile -t flags < <(schedule_flags "${file}")
85+
86+
io::run gcloud scheduler jobs create http "${id}" "${flags[@]}" \
87+
--project "${CLOUD_PROJECT}" \
88+
--location "${LOCATION}"
89+
}
90+
91+
# Use getopt to parse and normalize all the args.
92+
PARSED="$(getopt -a \
93+
--options="h" \
94+
--longoptions="list,ids,export:,import:,help" \
95+
--name="${PROGRAM_NAME}" \
96+
-- "$@")"
97+
eval set -- "${PARSED}"
98+
99+
case "$1" in
100+
--list)
101+
list_schedules
102+
;;
103+
--ids)
104+
ids_schedules
105+
;;
106+
--export)
107+
export_schedule "$2"
108+
;;
109+
--import)
110+
import_schedule "$2"
111+
;;
112+
-h | --help)
113+
print_usage
114+
;;
115+
*)
116+
print_usage
117+
exit 1
118+
;;
119+
esac
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"attemptDeadline": "180s",
3+
"description": "Daily Run",
4+
"httpTarget": {
5+
"headers": {
6+
"User-Agent": "Google-Cloud-Scheduler"
7+
},
8+
"httpMethod": "POST",
9+
"oauthToken": {
10+
"scope": "https://www.googleapis.com/auth/cloud-platform",
11+
"serviceAccountEmail": "cloud-build-trigger-scheduler@cloud-cpp-testing-resources.iam.gserviceaccount.com"
12+
},
13+
"uri": "https://cloudbuild.googleapis.com/v1/projects/cloud-cpp-testing-resources/locations/global/triggers/6d247368-98d8-4b06-9179-afeedd054f50:run"
14+
},
15+
"lastAttemptTime": "2023-09-27T21:43:00.097397Z",
16+
"name": "projects/cloud-cpp-testing-resources/locations/us-central1/jobs/clang-tidy-compute-schedule",
17+
"retryConfig": {
18+
"maxBackoffDuration": "3600s",
19+
"maxDoublings": 5,
20+
"maxRetryDuration": "0s",
21+
"minBackoffDuration": "5s"
22+
},
23+
"schedule": "43 21 * * *",
24+
"scheduleTime": "2023-09-28T21:43:00.460374Z",
25+
"state": "ENABLED",
26+
"status": {},
27+
"timeZone": "Etc/UTC",
28+
"userUpdateTime": "2023-09-27T14:55:05Z"
29+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"attemptDeadline": "180s",
3+
"httpTarget": {
4+
"headers": {
5+
"User-Agent": "Google-Cloud-Scheduler"
6+
},
7+
"httpMethod": "POST",
8+
"oauthToken": {
9+
"scope": "https://www.googleapis.com/auth/cloud-platform",
10+
"serviceAccountEmail": "cloud-build-trigger-scheduler@cloud-cpp-testing-resources.iam.gserviceaccount.com"
11+
},
12+
"uri": "https://cloudbuild.googleapis.com/v1/projects/cloud-cpp-testing-resources/locations/global/triggers/e21f4d09-213f-41ef-9be5-62309adf4287:run"
13+
},
14+
"lastAttemptTime": "2023-09-28T06:17:00.775386Z",
15+
"name": "projects/cloud-cpp-testing-resources/locations/us-central1/jobs/grpc-at-head-schedule",
16+
"retryConfig": {
17+
"maxBackoffDuration": "3600s",
18+
"maxDoublings": 5,
19+
"maxRetryDuration": "0s",
20+
"minBackoffDuration": "5s"
21+
},
22+
"schedule": "17 06 * * *",
23+
"scheduleTime": "2023-09-29T06:17:00.170313Z",
24+
"state": "ENABLED",
25+
"status": {},
26+
"timeZone": "Etc/UTC",
27+
"userUpdateTime": "2023-03-27T18:53:39Z"
28+
}
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"attemptDeadline": "180s",
3+
"httpTarget": {
4+
"body": "eyJicmFuY2hOYW1lIjoibWFpbiJ9",
5+
"headers": {
6+
"Content-Type": "application/octet-stream",
7+
"User-Agent": "Google-Cloud-Scheduler"
8+
},
9+
"httpMethod": "POST",
10+
"oauthToken": {
11+
"scope": "https://www.googleapis.com/auth/cloud-platform",
12+
"serviceAccountEmail": "cloud-build-trigger-scheduler@cloud-cpp-testing-resources.iam.gserviceaccount.com"
13+
},
14+
"uri": "https://cloudbuild.googleapis.com/v1/projects/cloud-cpp-testing-resources/triggers/03b9bb52-8d28-4095-9ca4-653f8602acf7:run"
15+
},
16+
"lastAttemptTime": "2023-09-28T07:00:00.004098Z",
17+
"name": "projects/cloud-cpp-testing-resources/locations/us-central1/jobs/integration-daily-schedule",
18+
"retryConfig": {
19+
"maxBackoffDuration": "3600s",
20+
"maxDoublings": 5,
21+
"maxRetryDuration": "0s",
22+
"minBackoffDuration": "5s"
23+
},
24+
"schedule": "0 3 * * *",
25+
"scheduleTime": "2023-09-29T07:00:00.794683Z",
26+
"state": "ENABLED",
27+
"status": {},
28+
"timeZone": "America/New_York",
29+
"userUpdateTime": "2021-05-26T20:01:26Z"
30+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"attemptDeadline": "180s",
3+
"httpTarget": {
4+
"headers": {
5+
"User-Agent": "Google-Cloud-Scheduler"
6+
},
7+
"httpMethod": "POST",
8+
"oauthToken": {
9+
"scope": "https://www.googleapis.com/auth/cloud-platform",
10+
"serviceAccountEmail": "cloud-build-trigger-scheduler@cloud-cpp-testing-resources.iam.gserviceaccount.com"
11+
},
12+
"uri": "https://cloudbuild.googleapis.com/v1/projects/cloud-cpp-testing-resources/locations/global/triggers/7dc9cea4-9d0b-4d0f-9947-5b79bfee5c12:run"
13+
},
14+
"lastAttemptTime": "2023-09-28T02:23:00.078719Z",
15+
"name": "projects/cloud-cpp-testing-resources/locations/us-central1/jobs/protobuf-at-head-schedule",
16+
"retryConfig": {
17+
"maxBackoffDuration": "3600s",
18+
"maxDoublings": 5,
19+
"maxRetryDuration": "0s",
20+
"minBackoffDuration": "5s"
21+
},
22+
"schedule": "23 2 * * *",
23+
"state": "PAUSED",
24+
"timeZone": "Etc/UTC",
25+
"userUpdateTime": "2023-09-28T13:26:17Z"
26+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"attemptDeadline": "180s",
3+
"description": "Runs rotate keys every day at 6am",
4+
"httpTarget": {
5+
"body": "eyJicmFuY2hOYW1lIjoibWFpbiJ9",
6+
"headers": {
7+
"Content-Type": "application/octet-stream",
8+
"User-Agent": "Google-Cloud-Scheduler"
9+
},
10+
"httpMethod": "POST",
11+
"oauthToken": {
12+
"scope": "https://www.googleapis.com/auth/cloud-platform",
13+
"serviceAccountEmail": "cloud-build-trigger-scheduler@cloud-cpp-testing-resources.iam.gserviceaccount.com"
14+
},
15+
"uri": "https://cloudbuild.googleapis.com/v1/projects/cloud-cpp-testing-resources/triggers/3fca51de-ec4c-453f-9693-5ccec556a379:run"
16+
},
17+
"lastAttemptTime": "2023-09-28T10:00:00.341621Z",
18+
"name": "projects/cloud-cpp-testing-resources/locations/us-central1/jobs/rotate-keys-schedule",
19+
"retryConfig": {
20+
"maxBackoffDuration": "3600s",
21+
"maxDoublings": 5,
22+
"maxRetryDuration": "0s",
23+
"minBackoffDuration": "5s"
24+
},
25+
"schedule": "0 6 * * *",
26+
"scheduleTime": "2023-09-29T10:00:00.165730Z",
27+
"state": "ENABLED",
28+
"status": {},
29+
"timeZone": "America/New_York",
30+
"userUpdateTime": "2021-05-26T19:52:35Z"
31+
}

0 commit comments

Comments
 (0)