Skip to content

Commit acc14a1

Browse files
serdarozerrserdarozerr
committed
feat: github workflow is added
- setup, run and teardown steps separated. - Makefile updated
1 parent f2cfffe commit acc14a1

File tree

6 files changed

+305
-0
lines changed

6 files changed

+305
-0
lines changed

‎.github/scripts/gcs/Makefile‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
SHELL = bash
15+
TMP_DIR := /tmp/storage-cli-gcs-$(or $(GITHUB_RUN_ID),$(USER))
16+
default: test-int
17+
18+
# Ensure tmp directory exists
19+
create-tmp-dir:
20+
@mkdir -p "$(TMP_DIR)"
21+
22+
# Generate a $StorageClass.lock which contains our bucket name
23+
# used for testing. Buckets must be unique among all in GCS,
24+
# we cannot simply hardcode a bucket.
25+
.PHONY: FORCE
26+
regional.lock: create-tmp-dir
27+
@test -s "$(TMP_DIR)/regional.lock" || \
28+
echo "gcs-$$(openssl rand -hex 20)" > "$(TMP_DIR)/regional.lock"
29+
30+
# Create a bucket using the name located in $StorageClass.lock with
31+
# a sane location.
32+
regional-bucket: regional.lock
33+
@gsutil ls | grep -q "$$(cat $(TMP_DIR)/regional.lock)"; if [ $$? -ne 0 ]; then \
34+
gsutil mb -c REGIONAL -l us-east1 "gs://$$(cat $(TMP_DIR)/regional.lock)"; \
35+
fi
36+
37+
.PHONY: FORCE
38+
multiregional.lock: create-tmp-dir
39+
@test -s "$(TMP_DIR)/multiregional.lock" || \
40+
echo "gcs-$$(openssl rand -hex 20)" > "$(TMP_DIR)/multiregional.lock"
41+
42+
multiregional-bucket: multiregional.lock
43+
@gsutil ls | grep -q "$$(cat $(TMP_DIR)/multiregional.lock)"; if [ $$? -ne 0 ]; then \
44+
gsutil mb -c MULTI_REGIONAL -l us "gs://$$(cat $(TMP_DIR)/multiregional.lock)"; \
45+
fi
46+
47+
.PHONY: FORCE
48+
public.lock: create-tmp-dir
49+
@test -s "$(TMP_DIR)/public.lock" || \
50+
echo "gcs-$$(openssl rand -hex 20)" > "$(TMP_DIR)/public.lock"
51+
52+
53+
public-bucket: public.lock
54+
@gsutil ls | grep -q "$$(cat $(TMP_DIR)/public.lock)"; if [ $$? -ne 0 ]; then \
55+
gsutil mb -c MULTI_REGIONAL -l us "gs://$$(cat $(TMP_DIR)/public.lock)" && \
56+
gsutil iam ch allUsers:legacyObjectReader "gs://$$(cat $(TMP_DIR)/public.lock)" && \
57+
gsutil iam ch allUsers:legacyBucketReader "gs://$$(cat $(TMP_DIR)/public.lock)" && \
58+
echo "waiting for IAM to propagate" && \
59+
until curl -s \
60+
"https://storage.googleapis.com/$$(cat $(TMP_DIR)/public.lock)/non-existent" \
61+
| grep -q "NoSuchKey"; do sleep 1; done; \
62+
fi
63+
64+
# Create all buckets necessary for the test.
65+
prep-gcs: regional-bucket multiregional-bucket public-bucket
66+
67+
# Remove all buckets listed in $StorageClass.lock files.
68+
clean-gcs:
69+
@test -s "$(TMP_DIR)/multiregional.lock" && test -s "$(TMP_DIR)/regional.lock" && test -s "$(TMP_DIR)/public.lock"
70+
@gsutil rm "gs://$$(cat $(TMP_DIR)/regional.lock)/*" || true
71+
@gsutil rb "gs://$$(cat $(TMP_DIR)/regional.lock)"
72+
@rm "$(TMP_DIR)/regional.lock"
73+
@gsutil rm "gs://$$(cat $(TMP_DIR)/multiregional.lock)/*" || true
74+
@gsutil rb "gs://$$(cat $(TMP_DIR)/multiregional.lock)"
75+
@rm "$(TMP_DIR)/multiregional.lock"
76+
@gsutil rm "gs://$$(cat $(TMP_DIR)/public.lock)/*" || true
77+
@gsutil rb "gs://$$(cat $(TMP_DIR)/public.lock)"
78+
@rm "$(TMP_DIR)/public.lock"
79+
@rmdir "$(TMP_DIR)" 2>/dev/null || true
80+
81+
# Perform only unit tests
82+
test-unit:
83+
cd ../../.. && go run github.com/onsi/ginkgo/v2/ginkgo -r --skip-package integration
84+
85+
# Perform all tests, including integration tests.
86+
test-int:
87+
export MULTIREGIONAL_BUCKET_NAME="$$(cat $(TMP_DIR)/multiregional.lock)" && \
88+
export REGIONAL_BUCKET_NAME="$$(cat $(TMP_DIR)/regional.lock)" && \
89+
export PUBLIC_BUCKET_NAME="$$(cat $(TMP_DIR)/public.lock)" && \
90+
cd ../../.. && go run github.com/onsi/ginkgo/v2/ginkgo gcs/integration/
91+
92+
# Perform all non-long tests, including integration tests.
93+
test-fast-int:
94+
export MULTIREGIONAL_BUCKET_NAME="$$(cat $(TMP_DIR)/multiregional.lock)" && \
95+
export REGIONAL_BUCKET_NAME="$$(cat $(TMP_DIR)/regional.lock)" && \
96+
export PUBLIC_BUCKET_NAME="$$(cat $(TMP_DIR)/public.lock)" && \
97+
export SKIP_LONG_TESTS="yes" && \
98+
cd ../../.. && go run github.com/onsi/ginkgo/v2/ginkgo gcs/integration/
99+
100+
help:
101+
@echo " prep-gcs: create external GCS buckets needed for integration testing"
102+
@echo " clean-gcs: remove external GCS buckets"
103+
@echo " test-fast-int: run an reduced integration test suite (presubmit)"
104+
@echo " test-int: run the full integration test (CI only)"
105+
@echo ""
106+
@echo "expected environment variables:"
107+
@echo " GOOGLE_SERVICE_ACCOUNT=contents of a JSON service account key"

‎.github/scripts/gcs/run-int.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
5+
# Get the directory where this script is located
6+
script_dir="$( cd "$(dirname "${0}")" && pwd )"
7+
repo_root="$(cd "${script_dir}/../../.." && pwd)"
8+
9+
: "${google_json_key_data:?}"
10+
11+
export GOOGLE_SERVICE_ACCOUNT="${google_json_key_data}"
12+
13+
pushd "${script_dir}" > /dev/null
14+
source utils.sh
15+
gcloud_login
16+
popd > /dev/null
17+
18+
pushd "${script_dir}"
19+
20+
# Set up conditional long test execution
21+
if [[ "${SKIP_LONG_TESTS:-}" == "yes" ]]; then
22+
make test-fast-int
23+
else
24+
make test-int
25+
fi
26+
popd
27+
28+

‎.github/scripts/gcs/setup.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
5+
# Get the directory where this script is located
6+
script_dir="$( cd "$(dirname "${0}")" && pwd )"
7+
repo_root="$(cd "${script_dir}/../../.." && pwd)"
8+
9+
10+
: "${google_json_key_data:?}"
11+
12+
pushd "${script_dir}"
13+
source utils.sh
14+
gcloud_login
15+
make prep-gcs
16+
popd

‎.github/scripts/gcs/teardown.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
5+
# Get the directory where this script is located
6+
script_dir="$( cd "$(dirname "${0}")" && pwd )"
7+
repo_root="$(cd "${script_dir}/../../.." && pwd)"
8+
9+
10+
: "${google_json_key_data:?}"
11+
12+
pushd "${script_dir}"
13+
source utils.sh
14+
gcloud_login
15+
make clean-gcs
16+
popd

‎.github/scripts/gcs/utils.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/usr/bin/env bash
2+
3+
check_param() {
4+
local name=$1
5+
local value
6+
value=$(eval echo '$'"${name}")
7+
if [ "${value}" == 'replace-me' ]; then
8+
echo "environment variable ${name} must be set"
9+
exit 1
10+
fi
11+
}
12+
13+
print_git_state() {
14+
echo "--> last commit..."
15+
TERM=xterm-256color git log -1
16+
echo "---"
17+
echo "--> local changes (e.g., from 'fly execute')..."
18+
TERM=xterm-256color git status --verbose
19+
echo "---"
20+
}
21+
22+
declare -a on_exit_items
23+
on_exit_items=()
24+
25+
function on_exit {
26+
echo "Running ${#on_exit_items[@]} on_exit items..."
27+
for i in "${on_exit_items[@]}"
28+
do
29+
for try in $(seq 0 9); do
30+
sleep "${try}"
31+
echo "Running cleanup command $i (try: ${try})"
32+
eval "${i}" || continue
33+
break
34+
done
35+
done
36+
}
37+
38+
function add_on_exit {
39+
local n=${#on_exit_items[@]}
40+
on_exit_items=("${on_exit_items[@]}" "$*")
41+
if [[ $n -eq 0 ]]; then
42+
trap on_exit EXIT
43+
fi
44+
}
45+
46+
function clean_gcs {
47+
pushd "${release_dir}" || return
48+
make clean-gcs
49+
popd || return
50+
}
51+
52+
function set_env {
53+
my_dir=$(dirname "$(readlink -f "${0}")")
54+
export release_dir
55+
release_dir="$( cd "${my_dir}" && cd ../.. && pwd )"
56+
export workspace_dir
57+
workspace_dir="$( cd "${release_dir}" && cd .. && pwd )"
58+
59+
go_bin=$(go env GOPATH)
60+
export PATH=${go_bin}/bin:${PATH}
61+
}
62+
63+
64+
function gcloud_login {
65+
check_param 'google_json_key_data'
66+
google_project=$(echo ${google_json_key_data} | jq -r .project_id)
67+
68+
keyfile=$(mktemp)
69+
echo "${google_json_key_data:-}" > "${keyfile}"
70+
gcloud auth activate-service-account --key-file="${keyfile}"
71+
gcloud config set project "${google_project:-}"
72+
}

‎.github/workflows/gcs-integration.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: GCS Integration Tests
2+
3+
on:
4+
push:
5+
pull_request:
6+
7+
jobs:
8+
gcs-integration-fast-tests:
9+
name: GCS Integation Fast Tests
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout code
13+
uses: actions/checkout@v5
14+
- name: Set up Go
15+
uses: actions/setup-go@v6
16+
with:
17+
go-version-file: go.mod
18+
- name: Install Ginkgo
19+
run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
20+
- name: Setup GCS Test Environment
21+
run: |
22+
echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" | base64 -d > /tmp/gcp-key.json
23+
export google_json_key_data="/tmp/gcp-key.json"
24+
./.github/scripts/gcs/setup.sh
25+
- name: Run Fast Tests
26+
run: |
27+
echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" | base64 -d > /tmp/gcp-key.json
28+
export google_json_key_data="/tmp/gcp-key.json"
29+
export SKIP_LONG_TESTS=yes
30+
./.github/scripts/gcs/run-int.sh
31+
- name: Teardown GCS Test Environment
32+
if: always()
33+
run: |
34+
echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" | base64 -d > /tmp/gcp-key.json
35+
export google_json_key_data="/tmp/gcp-key.json"
36+
./.github/scripts/gcs/teardown.sh
37+
38+
gcs-integration-all-tests:
39+
name: GCS Integation All Tests
40+
runs-on: ubuntu-latest
41+
steps:
42+
- name: Checkout code
43+
uses: actions/checkout@v5
44+
- name: Set up Go
45+
uses: actions/setup-go@v6
46+
with:
47+
go-version-file: go.mod
48+
- name: Install Ginkgo
49+
run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
50+
- name: Setup GCS Test Environment
51+
run: |
52+
echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" | base64 -d > /tmp/gcp-key.json
53+
export google_json_key_data="/tmp/gcp-key.json"
54+
./.github/scripts/gcs/setup.sh
55+
- name: Run All Tests
56+
run: |
57+
echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" | base64 -d > /tmp/gcp-key.json
58+
export google_json_key_data="/tmp/gcp-key.json"
59+
./.github/scripts/gcs/run-int.sh
60+
- name: Teardown GCS Test Environment
61+
if: always()
62+
run: |
63+
echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" | base64 -d > /tmp/gcp-key.json
64+
export google_json_key_data="/tmp/gcp-key.json"
65+
./.github/scripts/gcs/teardown.sh
66+

0 commit comments

Comments
 (0)