Skip to content

Commit 44f17f5

Browse files
daniel-sanchedaniel-sanche
authored
test: add performance tests (#595)
1 parent 44eb868 commit 44f17f5

File tree

5 files changed

+595
-0
lines changed

5 files changed

+595
-0
lines changed

‎.kokoro/performance/common.cfg‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
# Build logs will be here
4+
action {
5+
define_artifacts {
6+
regex: "**/*sponge_log.xml"
7+
}
8+
}
9+
10+
# Download trampoline resources.
11+
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
12+
13+
# Download resources for system tests (service account key, etc.)
14+
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python"
15+
16+
# Use the trampoline script to run in docker.
17+
build_file: "python-logging/.kokoro/trampoline.sh"
18+
19+
# Configure the docker image for kokoro-trampoline.
20+
env_vars: {
21+
key: "TRAMPOLINE_IMAGE"
22+
value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
23+
}
24+
env_vars: {
25+
key: "TRAMPOLINE_BUILD_FILE"
26+
value: "github/python-logging/.kokoro/test-performance.sh"
27+
}

‎.kokoro/performance/presubmit.cfg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto

‎.kokoro/test-performance.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
# Copyright 2022 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
17+
set -eox pipefail
18+
19+
if [[ -z "${PROJECT_ROOT:-}" ]]; then
20+
PROJECT_ROOT="github/python-logging"
21+
fi
22+
23+
cd "${PROJECT_ROOT}/tests/performance"
24+
25+
26+
# Disable buffering, so that the logs stream through.
27+
export PYTHONUNBUFFERED=1
28+
29+
# Debug: show build environment
30+
env | grep KOKORO
31+
32+
33+
# Install nox
34+
python3 -m pip install --upgrade --quiet nox
35+
36+
# run performance tests
37+
set +e
38+
python3 -m nox
39+
TEST_STATUS_CODE=$?
40+
41+
# Workaround for Kokoro permissions issue: delete secrets
42+
rm testing/{test-env.sh,client-secrets.json,service-account.json}
43+
44+
exit "$TEST_STATUS_CODE"

‎tests/performance/noxfile.py‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
from __future__ import absolute_import
18+
import os
19+
import pathlib
20+
import re
21+
from colorlog.escape_codes import parse_colors
22+
23+
import nox
24+
25+
26+
DEFAULT_PYTHON_VERSION = "3.8"
27+
28+
PERFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"]
29+
30+
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
31+
REPO_ROOT_DIRECTORY = CURRENT_DIRECTORY.parent.parent
32+
33+
REPO_URL = "https://github.com/googleapis/python-logging.git"
34+
CLONE_REPO_DIR = "python-logging-main"
35+
36+
# 'docfx' is excluded since it only needs to run in 'docs-presubmit'
37+
nox.options.sessions = ["performance", "performance_regression"]
38+
39+
# Error if a python version is missing
40+
nox.options.error_on_missing_interpreters = True
41+
42+
43+
@nox.session(python=PERFORMANCE_TEST_PYTHON_VERSIONS)
44+
def performance(session):
45+
"""Run the performance test suite."""
46+
# Use pre-release gRPC for performance tests.
47+
session.install("--pre", "grpcio")
48+
49+
# Install all test dependencies, then install this package into the
50+
# virtualenv's dist-packages.
51+
session.install(
52+
"mock",
53+
"pandas",
54+
"rich",
55+
"pytest",
56+
"google-cloud-testutils",
57+
)
58+
session.install("-e", str(REPO_ROOT_DIRECTORY))
59+
60+
file_path = f"perf_{session.python}_sponge_log.xml"
61+
session.run(
62+
"py.test",
63+
f"--ignore={CLONE_REPO_DIR}",
64+
"-s",
65+
f"--junitxml={file_path}",
66+
str(CURRENT_DIRECTORY),
67+
*session.posargs,
68+
)
69+
get_junitxml_results(file_path)
70+
71+
72+
@nox.session(python=PERFORMANCE_TEST_PYTHON_VERSIONS)
73+
def print_last_results(session):
74+
"""Print results from last performance test session."""
75+
file_path = f"perf_{session.python}_sponge_log.xml"
76+
get_junitxml_results(file_path)
77+
78+
79+
def get_junitxml_results(file_path, print_results=True):
80+
"""Print results from specified results file."""
81+
results = None
82+
if os.path.exists(file_path):
83+
if print_results:
84+
print(f"{file_path} results:")
85+
with open(file_path, "r") as file:
86+
data = file.read().replace("\n", "")
87+
total = 0
88+
results = {}
89+
for entry in data.split("testcase classname")[1:]:
90+
name = re.search(r'name="+(\w+)', entry)[1]
91+
time = re.search(r'time="+([0-9\.]+)', entry)[1]
92+
total += float(time)
93+
if print_results:
94+
print(f"\t{name}: {time}s")
95+
results[name] = float(time)
96+
if print_results:
97+
print(f"\tTotal: {total:.3f}s")
98+
else:
99+
print(f"error: {file_path} not found")
100+
return results
101+
102+
103+
@nox.session(python=PERFORMANCE_TEST_PYTHON_VERSIONS)
104+
def performance_regression(session, percent_threshold=10):
105+
"""Check performance against repo main."""
106+
107+
clone_dir = os.path.join(CURRENT_DIRECTORY, CLONE_REPO_DIR)
108+
109+
if not os.path.exists(clone_dir):
110+
print("downloading copy of repo at `main`")
111+
session.run("git", "clone", REPO_URL, CLONE_REPO_DIR)
112+
113+
# Use pre-release gRPC for performance tests.
114+
session.install("--pre", "grpcio")
115+
116+
# Install all test dependencies, then install this package into the
117+
# virtualenv's dist-packages.
118+
session.install(
119+
"mock",
120+
"pandas",
121+
"rich",
122+
"pytest",
123+
"google-cloud-testutils",
124+
)
125+
126+
main_file_name = f"main_perf_{session.python}_sponge_log.xml"
127+
head_file_name = f"head_perf_{session.python}_sponge_log.xml"
128+
# test against main
129+
print("testing against library at `main`...")
130+
session.install("-e", str(clone_dir))
131+
session.run(
132+
"py.test",
133+
f"--ignore={CLONE_REPO_DIR}",
134+
"-s",
135+
f"--junitxml={main_file_name}",
136+
str(CURRENT_DIRECTORY),
137+
*session.posargs,
138+
success_codes=[1, 0], # don't report failures at this step
139+
)
140+
# test head
141+
print("testing against library at `HEAD`...")
142+
session.install("-e", str(REPO_ROOT_DIRECTORY))
143+
session.run(
144+
"py.test",
145+
f"--ignore={CLONE_REPO_DIR}",
146+
"-s",
147+
f"--junitxml={head_file_name}",
148+
str(CURRENT_DIRECTORY),
149+
*session.posargs,
150+
success_codes=[1, 0], # don't report failures at this step
151+
)
152+
# print results
153+
main_results = get_junitxml_results(main_file_name, print_results=False)
154+
head_results = get_junitxml_results(head_file_name, print_results=False)
155+
all_pass = True
156+
for test, time in head_results.items():
157+
if test in main_results:
158+
prev_time = main_results[test]
159+
diff = time - prev_time
160+
percent_diff = diff / prev_time
161+
test_passes = percent_diff * 100 < percent_threshold
162+
all_pass = all_pass and test_passes
163+
if not test_passes:
164+
color = parse_colors("red")
165+
elif diff > 0:
166+
color = parse_colors("yellow")
167+
else:
168+
color = parse_colors("green")
169+
print(
170+
f"{test}: {color} {diff:+.3f}s ({percent_diff:+.1%}){parse_colors('reset')}"
171+
)
172+
else:
173+
print(f"{test}: ???")
174+
if not all_pass:
175+
session.error(f"performance degraded >{percent_threshold}%")

0 commit comments

Comments
 (0)