Skip to content

Commit 3e4212a

Browse files
general-kroll-4-lifegeneral-kroll-4-life
committed
advanced-regression-testing
Summary: - Support for mocked testing. - Support for proxied mocked testing. - Support for `any-sdk` CLI testing. - Support for readonly live testing. - Support for readwrite live testing on tag **only**. - Traffic lights report.
1 parent aa53126 commit 3e4212a

28 files changed

+735
-44
lines changed

‎.github/workflows/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ jobs:
5757
5858
- name: "[SETUP] get updated providers"
5959
run: |
60-
pip install -q -r scripts/setup/requirements.txt
60+
pip install -q -r requirements.txt
6161
python scripts/setup/get-updated-providers.py
6262
6363
- name: "[SETUP] prepare dist and test dirs"

‎.github/workflows/regression.yml‎

Lines changed: 144 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
name: Integration Testing and Analysis
22

33
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- dev
48
push:
9+
branches:
10+
- main
11+
- dev
512
tags:
613
- robot*
714
- regression*
@@ -15,7 +22,7 @@ env:
1522
STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }}
1623

1724
jobs:
18-
build-and-deploy:
25+
regression-testing:
1926
name: build-and-deploy
2027
runs-on: ubuntu-latest
2128
permissions:
@@ -79,15 +86,36 @@ jobs:
7986
go get ./...
8087
python3 cicd/python/build.py --build
8188
89+
- name: Build any-sdk cli from source
90+
working-directory: stackql-any-sdk
91+
run: |
92+
93+
go get ./...
94+
95+
go build -x -v \
96+
-o build/anysdk ./cmd/interrogate
97+
8298
- name: Parse tag
8399
id: parse_tag
84100
run: |
85-
tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)"
86-
echo "tag_obj: $tag_obj"
87-
{
88-
echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')"
89-
echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')"
90-
} | tee -a "$GITHUB_ENV"
101+
if [ "${{ github.ref_type }}" = "tag" ]; then
102+
tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)"
103+
echo "tag_obj: $tag_obj"
104+
{
105+
echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')"
106+
echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')"
107+
} | tee -a "$GITHUB_ENV"
108+
else
109+
{
110+
echo "IS_BRANCH=true"
111+
} >> $GITHUB_ENV
112+
fi
113+
114+
115+
- name: Generate rewritten registry for simulations
116+
working-directory: stackql-core
117+
run: |
118+
python3 test/python/registry-rewrite.py
91119
92120
93121
- name: Prepare load balancing materials
@@ -118,8 +146,36 @@ jobs:
118146
openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
119147
openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
120148
openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365
149+
150+
151+
- name: Start Core Test Mocks
152+
working-directory: stackql-core
153+
run: |
154+
pgrep -f flask | xargs kill -9 || true
155+
flask --app=./test/python/flask/gcp/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 &
156+
flask --app=./test/python/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 &
157+
158+
- name: Run any-sdk cli mocked testing
159+
working-directory: stackql-core
160+
run: |
161+
export GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)"
162+
bucketsListIDs="$(${{ github.workspace }}/stackql-any-sdk/build/anysdk query \
163+
--svc-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/services/storage-v1.yaml" \
164+
--tls.allowInsecure \
165+
--prov-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/provider.yaml" \
166+
--resource buckets \
167+
--method list \
168+
--parameters '{ "project": "stackql-demo" }' \
169+
| jq -r '.items[].id')"
170+
matchingBuckets="$(echo "${bucketsListIDs}" | grep "stackql-demo" )"
171+
if [ "${matchingBuckets}" = "" ]; then
172+
echo "Core Test Failed with no matching buckets"
173+
exit 1
174+
else
175+
echo "Core Test passed with matching buckets: $matchingBuckets"
176+
fi
121177
122-
- name: Run core robot functional tests
178+
- name: Run core proxied robot functional tests against local registry
123179
if: success()
124180
working-directory: stackql-core
125181
run: |
@@ -130,12 +186,21 @@ jobs:
130186
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
131187
--include registry \
132188
-d test/robot/reports \
133-
test/robot/functional
189+
test/robot/functional || true
134190
135-
- name: Output from functional tests
191+
- name: Output from core proxied functional tests
136192
if: always()
137193
run: |
138194
cat stackql-core/test/robot/reports/output.xml
195+
python3 scripts/cicd/python/robot-parse.py --robot-output-file stackql-core/test/robot/reports/output.xml > stackql-core/test/robot/reports/proxied_parsed_output.json
196+
197+
- name: Upload core traffic lights
198+
uses: actions/[email protected]
199+
if: success()
200+
with:
201+
name: proxied-core-traffic-lights
202+
path: stackql-core/test/robot/reports/proxied_parsed_output.json
203+
139204

140205
- name: Post core test cleanup
141206
run: |
@@ -149,13 +214,21 @@ jobs:
149214
robot \
150215
--variable "${sundryCfg}" \
151216
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
152-
-d test/robot/reports \
153-
test/robot/stackql/mocked
217+
-d test/robot/reports/mocked \
218+
test/robot/stackql/mocked || true
154219
155220
- name: Output from local registry mocked functional tests
156221
if: always()
157222
run: |
158-
cat test/robot/reports/output.xml
223+
cat test/robot/reports/mocked/output.xml
224+
python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/mocked/output.xml > test/robot/reports/mocked/parsed_output.json
225+
226+
- name: Upload local registry mocked traffic lights
227+
uses: actions/[email protected]
228+
if: success()
229+
with:
230+
name: local-registry-mocked-traffic-lights
231+
path: test/robot/reports/mocked/parsed_output.json
159232

160233
- name: Post registry mocked test cleanup
161234
run: |
@@ -164,23 +237,74 @@ jobs:
164237
sudo cp /etc/hosts.bak /etc/hosts || true
165238
rm -f test/robot/reports/*.xml || true
166239
167-
- name: Run live robot functional tests
240+
- name: Run live readonly robot functional tests
168241
if: success()
169-
id: live_integration_tests
242+
id: live_integration_tests_readonly
170243
env:
171244
GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RO_SECRET }}
172245
AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
173246
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
174247
run: |
175248
providerRoot="$(realpath $(pwd)/providers)"
176-
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
249+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}'
177250
robot \
178251
--variable "${sundryCfg}" \
179252
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
180-
-d test/robot/reports \
181-
test/robot/stackql/live
253+
-d test/robot/reports/readonly \
254+
test/robot/stackql/live/readonly || true
182255
183-
- name: Output from live functional tests
256+
- name: Output from live readonly functional tests
184257
if: always()
185258
run: |
186-
cat test/robot/reports/output.xml
259+
cat test/robot/reports/readonly/output.xml
260+
python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/readonly/output.xml > test/robot/reports/readonly/parsed_output.json
261+
262+
- name: Upload readonly traffic lights
263+
uses: actions/[email protected]
264+
if: success()
265+
with:
266+
name: local-registry-readonly-traffic-lights
267+
path: test/robot/reports/readonly/parsed_output.json
268+
269+
- name: Run live readwrite robot functional tests
270+
if: github.ref_type == 'tag'
271+
id: live_integration_tests_readwrite
272+
env:
273+
GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RW_SECRET }}
274+
AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
275+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
276+
run: |
277+
providerRoot="$(realpath $(pwd)/providers)"
278+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}'
279+
robot \
280+
--variable "${sundryCfg}" \
281+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
282+
-d test/robot/reports/readwrite \
283+
test/robot/stackql/live/readwrite || true
284+
285+
- name: Output from live readwrite functional tests
286+
if: github.ref_type == 'tag'
287+
run: |
288+
cat test/robot/reports/readwrite/output.xml
289+
python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/readwrite/output.xml > test/robot/reports/readwrite/parsed_output.json
290+
291+
- name: Upload readonly traffic lights
292+
uses: actions/[email protected]
293+
if: success()
294+
with:
295+
name: local-registry-readwrite-traffic-lights
296+
path: test/robot/reports/readwrite/parsed_output.json
297+
298+
- name: Display traffic lights
299+
run: |
300+
for i in $(ls test/robot/reports/*/parsed_output.json); do
301+
echo "Traffic light for $i"
302+
if [ -f "$i" ]; then
303+
python3 scripts/cicd/python/display-parsed.py --traffic-light-file $i
304+
else
305+
echo "File $i does not exist 🛑"
306+
fi
307+
done
308+
echo "Traffic light for proxied"
309+
python3 scripts/cicd/python/display-parsed.py --traffic-light-file stackql-core/test/robot/reports/proxied_parsed_output.json
310+
echo "traffic lights completed"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ stackql-zip
88
.stackql/
99
stackql-core/
1010
stackql-any-sdk/
11+
.venv/

‎requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
boto3>=1.37.24
2+
PyYaml>=6.0.1
3+
requests==2.32.3
4+
robotframework==6.1.1

‎scripts/cicd/python/display-parsed.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import json
2+
3+
from argparse import ArgumentParser, Namespace
4+
5+
from typing import Iterable
6+
7+
def parse_args() -> Namespace:
8+
parser = ArgumentParser(description="Parse Robot Framework output XML file.")
9+
parser.add_argument(
10+
"--traffic-light-file",
11+
help="Path to the traffic lights json file.",
12+
)
13+
return parser.parse_args()
14+
15+
class _TrafficLightDisplayer(object):
16+
17+
_EMOJI_MAP = {
18+
"red": "🛑",
19+
"yellow": "🟡",
20+
"orange": "🟠",
21+
"grey": "⚪",
22+
"blue": "🔵",
23+
"green": "🟢",
24+
}
25+
26+
def __init__(self, traffic_light_file: str):
27+
self._traffic_light_file = traffic_light_file
28+
with open(traffic_light_file, "r") as f:
29+
data = json.load(f)
30+
self._data = data
31+
32+
def _display(self, key: str, traffic_light :str) -> str:
33+
return f'{key}: {self._EMOJI_MAP[traffic_light]}'
34+
35+
def _simple_assemble(self) -> Iterable[str]:
36+
result = []
37+
for key, value in self._data.get('tags', {}).items():
38+
result.append(f'Tag: {self._display(key, value)}')
39+
result.append(f'Total: {self._display("total", self._data.get("total"))}')
40+
return result
41+
42+
def render(self) -> None:
43+
result = self._simple_assemble()
44+
print(f'Traffic Light Summary for file: {self._traffic_light_file}')
45+
print(" ")
46+
print("\n ".join(result))
47+
print("")
48+
49+
50+
def main() -> None:
51+
args = parse_args()
52+
displayer = _TrafficLightDisplayer(args.traffic_light_file)
53+
displayer.render()
54+
55+
if __name__ == "__main__":
56+
main()
57+
58+

0 commit comments

Comments
 (0)