Skip to content

Commit 16e0bbc

Browse files
ameukamameukam
committed
GCP: Enable Artifact Registry for the Boskos projects
Ensure the service is enabled for all the Boskos projects. It's currently not consistent across the entire list. Also lint the bash script. Signed-off-by: Arnaud Meukam <[email protected]>
1 parent 2d9f68b commit 16e0bbc

File tree

1 file changed

+156
-155
lines changed

1 file changed

+156
-155
lines changed

‎infra/gcp/bash/prow/ensure-e2e-projects.sh

Lines changed: 156 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
2525
. "${SCRIPT_DIR}/../lib.sh"
2626

2727
function usage() {
28-
echo "usage: $0 [repo...]" > /dev/stderr
29-
echo "example:" > /dev/stderr
30-
echo " $0 # do all projects" > /dev/stderr
31-
echo " $0 k8s-infra-node-e2e-project # just do one" > /dev/stderr
32-
echo > /dev/stderr
28+
echo "usage: $0 [repo...]" >/dev/stderr
29+
echo "example:" >/dev/stderr
30+
echo " $0 # do all projects" >/dev/stderr
31+
echo " $0 k8s-infra-node-e2e-project # just do one" >/dev/stderr
32+
echo >/dev/stderr
3333
}
3434

3535
## projects hosting prow build clusters managed by sig-k8s-infra
@@ -47,170 +47,171 @@ mapfile -t E2E_PROJECTS < <(k8s_infra_projects "e2e")
4747
readonly E2E_PROJECTS
4848

4949
function ensure_e2e_project() {
50-
if [ $# != 1 ] || [ -z "$1" ]; then
51-
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
52-
return 1
53-
fi
54-
local prj="${1}"
55-
56-
ensure_project "${prj}"
57-
58-
local project_number
59-
project_number=$(gcloud projects describe "${prj}" --format='value(projectNumber)')
60-
61-
color 6 "Ensure stale role bindings have been removed from e2e project: ${prj}"
62-
(
63-
echo "no stale bindings slated for removal"
64-
) 2>&1 | indent
65-
66-
color 6 "Ensuring only APIs necessary for kubernetes e2e jobs to use e2e project: ${prj}"
67-
ensure_only_services "${prj}" \
68-
cloudkms.googleapis.com \
69-
compute.googleapis.com \
70-
container.googleapis.com \
71-
containerregistry.googleapis.com \
72-
file.googleapis.com \
73-
logging.googleapis.com \
74-
monitoring.googleapis.com \
75-
storage-component.googleapis.com
76-
77-
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
78-
# could consider using a more limited custom IAM role instead
79-
color 6 "Empower prow-build service account to edit e2e project: ${prj}"
80-
ensure_project_role_binding "${prj}" \
81-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
82-
"roles/editor"
83-
84-
# TODO: Remove this binding and clean up permissions in projects
85-
# This permission is superseded by roles/cloudkms.admin below
86-
# Ensure GCP CSI driver tests can manage KMS keys
87-
ensure_project_role_binding "${prj}" \
88-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
89-
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
90-
91-
# Ensure GCP Default Compute Service Account can administer KMS keys
92-
ensure_project_role_binding "${prj}" \
93-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
94-
"roles/cloudkms.admin"
95-
96-
# TODO: Remove this binding and clean up permissions in projects
97-
# Ensure GCP Default Compute Service Account can manage KMS keys
98-
ensure_project_role_binding "${prj}" \
99-
"serviceAccount:${project_number}[email protected]" \
100-
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
101-
102-
# Ensure GCP Default Compute Engine Service Agent Account can manage KMS
103-
# keys
104-
ensure_project_role_binding "${prj}" \
105-
"serviceAccount:service-${project_number}@compute-system.iam.gserviceaccount.com" \
106-
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
107-
108-
# TODO: Remove this binding and clean up permissions in projects
109-
# Ensure GCP CSI driver tests can use prow-build service account to
110-
# act as all other service accounts (eg: Compute Engine default service account)
111-
ensure_project_role_binding "${prj}" \
112-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
113-
"roles/iam.serviceAccountUser"
114-
115-
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
116-
# could consider using a more limited custom IAM role instead
117-
color 6 "Empower boskos-janitor service account to clean e2e project: ${prj}"
118-
ensure_project_role_binding "${prj}" \
119-
"serviceAccount:${BOSKOS_JANITOR_SVCACCT}" \
120-
"roles/editor"
121-
122-
color 6 "Empower [email protected] to admin e2e project: ${prj}"
50+
if [ $# != 1 ] || [ -z "$1" ]; then
51+
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
52+
return 1
53+
fi
54+
local prj="${1}"
55+
56+
ensure_project "${prj}"
57+
58+
local project_number
59+
project_number=$(gcloud projects describe "${prj}" --format='value(projectNumber)')
60+
61+
color 6 "Ensure stale role bindings have been removed from e2e project: ${prj}"
62+
(
63+
echo "no stale bindings slated for removal"
64+
) 2>&1 | indent
65+
66+
color 6 "Ensuring only APIs necessary for kubernetes e2e jobs to use e2e project: ${prj}"
67+
ensure_only_services "${prj}" \
68+
artifactregistry.googleapis.com \
69+
cloudkms.googleapis.com \
70+
compute.googleapis.com \
71+
container.googleapis.com \
72+
containerregistry.googleapis.com \
73+
file.googleapis.com \
74+
logging.googleapis.com \
75+
monitoring.googleapis.com \
76+
storage-component.googleapis.com
77+
78+
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
79+
# could consider using a more limited custom IAM role instead
80+
color 6 "Empower prow-build service account to edit e2e project: ${prj}"
81+
ensure_project_role_binding "${prj}" \
82+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
83+
"roles/editor"
84+
85+
# TODO: Remove this binding and clean up permissions in projects
86+
# This permission is superseded by roles/cloudkms.admin below
87+
# Ensure GCP CSI driver tests can manage KMS keys
88+
ensure_project_role_binding "${prj}" \
89+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
90+
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
91+
92+
# Ensure GCP Default Compute Service Account can administer KMS keys
93+
ensure_project_role_binding "${prj}" \
94+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
95+
"roles/cloudkms.admin"
96+
97+
# TODO: Remove this binding and clean up permissions in projects
98+
# Ensure GCP Default Compute Service Account can manage KMS keys
99+
ensure_project_role_binding "${prj}" \
100+
"serviceAccount:${project_number}[email protected]" \
101+
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
102+
103+
# Ensure GCP Default Compute Engine Service Agent Account can manage KMS
104+
# keys
105+
ensure_project_role_binding "${prj}" \
106+
"serviceAccount:service-${project_number}@compute-system.iam.gserviceaccount.com" \
107+
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
108+
109+
# TODO: Remove this binding and clean up permissions in projects
110+
# Ensure GCP CSI driver tests can use prow-build service account to
111+
# act as all other service accounts (eg: Compute Engine default service account)
112+
ensure_project_role_binding "${prj}" \
113+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
114+
"roles/iam.serviceAccountUser"
115+
116+
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
117+
# could consider using a more limited custom IAM role instead
118+
color 6 "Empower boskos-janitor service account to clean e2e project: ${prj}"
119+
ensure_project_role_binding "${prj}" \
120+
"serviceAccount:${BOSKOS_JANITOR_SVCACCT}" \
121+
"roles/editor"
122+
123+
color 6 "Empower [email protected] to admin e2e project: ${prj}"
124+
ensure_project_role_binding "${prj}" \
125+
"group:[email protected]" \
126+
"roles/owner"
127+
128+
# NB: prow.viewer role is defined in ensure-organization.sh, that needs to have been run first
129+
color 6 "Empower [email protected] to view specific resources in e2e project: ${prj}"
130+
ensure_project_role_binding "${prj}" \
131+
"group:[email protected]" \
132+
"$(custom_org_role_name "prow.viewer")"
133+
134+
if [[ "${prj}" =~ k8s-infra-e2e.*scale ]]; then
135+
color 6 "Empower [email protected] to admin e2e project: ${prj}"
123136
ensure_project_role_binding "${prj}" \
124-
"group:k8s-infra-prow[email protected]" \
137+
"group:k8s-infra-sig-scalability[email protected]" \
125138
"roles/owner"
139+
fi
140+
141+
color 6 "Ensure prow-build prowjobs are able to ssh to instances in e2e project: ${prj}"
142+
prow_build_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmYxHh/wwcV0P1aChuFLpl28w6DFyc7G5Xrw1F8wH1Re9AdxyemM2bTZ/PhsP3u9VDnNbyOw3UN00VFdumkFLjLf1WQ7Q6rZDlPjlw7urBIvAMqUecY6ae1znqsZ0dMBxOuPXHznlnjLjM5b7O7q5WsQMCA9Szbmz6DsuSyCuX0It2osBTN+8P/Fa6BNh3W8AF60M7L8/aUzLfbXVS2LIQKAHHD8CWqvXhLPuTJ03iSwFvgtAK1/J2XJwUP+OzAFrxj6A9LW5ZZgk3R3kRKr0xT/L7hga41rB1qy8Uz+Xr/PTVMNGW+nmU4bPgFchCK0JBK7B12ZcdVVFUEdpaAiKZ prow"
143+
k8s_prow_builds_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+/ZdafYYrJknk08g98sYS1Nr+aVdAnhHpQyXBx7EAT9pazCGaoiYnXgC82FAfTVMqdsqnIiP+7FgQTFLNYvBt8KsBd9qCkuMh/Q1QYVh4kfjjuGUrjfo020pxGSvp+67kbxm6lubaio9AgJ9XXE+SP1AYbyKTvXEzk5Tu7gGnRt3OrjVB+9eqTnVJOjS/BAOTJV5DWQ7xMubHlT9NmQ/S2hotMoiJJybYGUalOfcf8ZkyspU2oR+x13DCfjvFdzF4U0fb/uvTJZeu22w887M5y0YQulFY2LIeoAUE4XwoOv0nxzwbtZpqPHwtfLgq3G906KHW5e6slXu8kGda656n prow"
144+
ssh_keys_expected=(
145+
"${k8s_prow_builds_ssh_pubkey}"
146+
"${prow_build_ssh_pubkey}"
147+
# TODO(amwat,spiffxp): something is adding an extra prow: prefix, it is
148+
# unclear where in prow->kubetest2->cluster/log-dump.sh->`gcloud ssh`
149+
# this is happening
150+
"prow:${k8s_prow_builds_ssh_pubkey}"
151+
"prow:${prow_build_ssh_pubkey}"
152+
)
153+
154+
# append to project-wide ssh-keys metadata if not present
155+
ssh_keys_before="${TMPDIR}/ssh-keys.before.txt"
156+
ssh_keys_after="${TMPDIR}/ssh-keys.after.txt"
157+
gcloud compute project-info describe --project="${prj}" \
158+
--format='value(commonInstanceMetadata.items.filter(key:ssh-keys).extract(value).flatten())' |
159+
sed -e '/^$/d' >"${ssh_keys_before}"
160+
161+
cp "${ssh_keys_before}" "${ssh_keys_after}"
162+
163+
if [ "${K8S_INFRA_ENSURE_E2E_PROJECTS_RESETS_SSH_KEYS:-"false"}" == "true" ]; then
164+
printf '%s\n' "${ssh_keys_expected[@]}" >"${ssh_keys_after}"
165+
else
166+
for ssh_key in "${ssh_keys_expected[@]}"; do
167+
if ! grep -q "${ssh_key}" "${ssh_keys_before}"; then
168+
echo "${ssh_key}" >>"${ssh_keys_after}"
169+
fi
170+
done
171+
fi
126172

127-
# NB: prow.viewer role is defined in ensure-organization.sh, that needs to have been run first
128-
color 6 "Empower [email protected] to view specific resources in e2e project: ${prj}"
129-
ensure_project_role_binding "${prj}" \
130-
"group:[email protected]" \
131-
"$(custom_org_role_name "prow.viewer")"
132-
133-
if [[ "${prj}" =~ k8s-infra-e2e.*scale ]]; then
134-
color 6 "Empower [email protected] to admin e2e project: ${prj}"
135-
ensure_project_role_binding "${prj}" \
136-
"group:[email protected]" \
137-
"roles/owner"
138-
fi
139-
140-
color 6 "Ensure prow-build prowjobs are able to ssh to instances in e2e project: ${prj}"
141-
prow_build_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmYxHh/wwcV0P1aChuFLpl28w6DFyc7G5Xrw1F8wH1Re9AdxyemM2bTZ/PhsP3u9VDnNbyOw3UN00VFdumkFLjLf1WQ7Q6rZDlPjlw7urBIvAMqUecY6ae1znqsZ0dMBxOuPXHznlnjLjM5b7O7q5WsQMCA9Szbmz6DsuSyCuX0It2osBTN+8P/Fa6BNh3W8AF60M7L8/aUzLfbXVS2LIQKAHHD8CWqvXhLPuTJ03iSwFvgtAK1/J2XJwUP+OzAFrxj6A9LW5ZZgk3R3kRKr0xT/L7hga41rB1qy8Uz+Xr/PTVMNGW+nmU4bPgFchCK0JBK7B12ZcdVVFUEdpaAiKZ prow"
142-
k8s_prow_builds_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+/ZdafYYrJknk08g98sYS1Nr+aVdAnhHpQyXBx7EAT9pazCGaoiYnXgC82FAfTVMqdsqnIiP+7FgQTFLNYvBt8KsBd9qCkuMh/Q1QYVh4kfjjuGUrjfo020pxGSvp+67kbxm6lubaio9AgJ9XXE+SP1AYbyKTvXEzk5Tu7gGnRt3OrjVB+9eqTnVJOjS/BAOTJV5DWQ7xMubHlT9NmQ/S2hotMoiJJybYGUalOfcf8ZkyspU2oR+x13DCfjvFdzF4U0fb/uvTJZeu22w887M5y0YQulFY2LIeoAUE4XwoOv0nxzwbtZpqPHwtfLgq3G906KHW5e6slXu8kGda656n prow"
143-
ssh_keys_expected=(
144-
"${k8s_prow_builds_ssh_pubkey}"
145-
"${prow_build_ssh_pubkey}"
146-
# TODO(amwat,spiffxp): something is adding an extra prow: prefix, it is
147-
# unclear where in prow->kubetest2->cluster/log-dump.sh->`gcloud ssh`
148-
# this is happening
149-
"prow:${k8s_prow_builds_ssh_pubkey}"
150-
"prow:${prow_build_ssh_pubkey}"
151-
)
152-
153-
# append to project-wide ssh-keys metadata if not present
154-
ssh_keys_before="${TMPDIR}/ssh-keys.before.txt"
155-
ssh_keys_after="${TMPDIR}/ssh-keys.after.txt"
156-
gcloud compute project-info describe --project="${prj}" \
157-
--format='value(commonInstanceMetadata.items.filter(key:ssh-keys).extract(value).flatten())' \
158-
| sed -e '/^$/d' > "${ssh_keys_before}"
159-
160-
cp "${ssh_keys_before}" "${ssh_keys_after}"
161-
162-
if [ "${K8S_INFRA_ENSURE_E2E_PROJECTS_RESETS_SSH_KEYS:-"false"}" == "true" ]; then
163-
printf '%s\n' "${ssh_keys_expected[@]}" > "${ssh_keys_after}"
164-
else
165-
for ssh_key in "${ssh_keys_expected[@]}"; do
166-
if ! grep -q "${ssh_key}" "${ssh_keys_before}"; then
167-
echo "${ssh_key}" >> "${ssh_keys_after}"
168-
fi
169-
done
170-
fi
171-
172-
if ! diff "${ssh_keys_before}" "${ssh_keys_after}" >/dev/null; then
173-
gcloud compute project-info add-metadata --project="${prj}" \
174-
--metadata-from-file ssh-keys="${ssh_keys_after}"
175-
diff_colorized "${ssh_keys_before}" "${ssh_keys_after}"
176-
fi
173+
if ! diff "${ssh_keys_before}" "${ssh_keys_after}" >/dev/null; then
174+
gcloud compute project-info add-metadata --project="${prj}" \
175+
--metadata-from-file ssh-keys="${ssh_keys_after}"
176+
diff_colorized "${ssh_keys_before}" "${ssh_keys_after}"
177+
fi
177178
}
178179

179180
# Disable OS Login at the project level
180181
# $1 The GCP Project
181182
function disable_project_oslogin() {
182-
if [ $# != 1 ] || [ -z "$1" ]; then
183-
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
184-
return 1
185-
fi
186-
187-
local prj="${1}"
188-
189-
enabled=$(gcloud compute project-info describe --project="${prj}" \
190-
--format='value(commonInstanceMetadata.items[enable-oslogin])')
191-
if [ "${enabled}" == "TRUE" ]; then
192-
gcloud compute project-info --project="${prj}" remove-metadata --keys "enable-oslogin"
193-
fi
183+
if [ $# != 1 ] || [ -z "$1" ]; then
184+
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
185+
return 1
186+
fi
187+
188+
local prj="${1}"
189+
190+
enabled=$(gcloud compute project-info describe --project="${prj}" \
191+
--format='value(commonInstanceMetadata.items[enable-oslogin])')
192+
if [ "${enabled}" == "TRUE" ]; then
193+
gcloud compute project-info --project="${prj}" remove-metadata --keys "enable-oslogin"
194+
fi
194195
}
195196

196197
function ensure_e2e_projects() {
197-
# default to all staging projects
198-
if [ $# = 0 ]; then
199-
set -- "${E2E_PROJECTS[@]}"
198+
# default to all staging projects
199+
if [ $# = 0 ]; then
200+
set -- "${E2E_PROJECTS[@]}"
201+
fi
202+
203+
for project in "${@}"; do
204+
if ! (printf '%s\n' "${E2E_PROJECTS[@]}" | grep -q "^${project}$"); then
205+
color 2 "Skipping unrecognized e2e project name: ${project}"
206+
continue
200207
fi
201208

202-
for project in "${@}"; do
203-
if ! (printf '%s\n' "${E2E_PROJECTS[@]}" | grep -q "^${project}$"); then
204-
color 2 "Skipping unrecognized e2e project name: ${project}"
205-
continue
206-
fi
207-
208-
color 3 "Configuring e2e project: ${project}"
209-
ensure_e2e_project "${project}" 2>&1 | indent
209+
color 3 "Configuring e2e project: ${project}"
210+
ensure_e2e_project "${project}" 2>&1 | indent
210211

211-
# color 3 "Ensuring OS Login is disabled for $project"
212-
# disable_project_oslogin "${project}" 2>&1 | indent
213-
done
212+
# color 3 "Ensuring OS Login is disabled for $project"
213+
# disable_project_oslogin "${project}" 2>&1 | indent
214+
done
214215
}
215216

216217
#

0 commit comments

Comments
 (0)