Skip to content

Commit c053b2e

Browse files
fcostaoliveirafcostaoliveira
committed
Fixed src build and docker logs retrieval in case of error
1 parent f5d9bc4 commit c053b2e

File tree

7 files changed

+116
-38
lines changed

7 files changed

+116
-38
lines changed

‎pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redis-benchmarks-specification"
3-
version = "0.1.215"
3+
version = "0.1.216"
44
description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
55
authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
66
readme = "Readme.md"

‎redis_benchmarks_specification/__builder__/builder.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,8 @@ def builder_process_stream(
343343
"linenoise",
344344
"lua",
345345
]
346+
if "fast_float" in deps_dir:
347+
deps_list.append("fast_float")
346348
if "hdr_histogram" in deps_dir:
347349
deps_list.append("hdr_histogram")
348350
if "fpconv" in deps_dir:

‎redis_benchmarks_specification/__self_contained_coordinator__/docker.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def generate_standalone_redis_server_args(
2020
"{}".format(port),
2121
]
2222
if dbdir != "":
23-
command.extend(["--dbdir", dbdir])
23+
command.extend(["--dir", dbdir])
2424
if configuration_parameters is not None:
2525
for parameter, parameter_value in configuration_parameters.items():
2626
if parameter not in added_params:

‎redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

Lines changed: 55 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -832,42 +832,16 @@ def process_self_contained_coordinator_stream(
832832
db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
833833
ceil_db_cpu_limit, current_cpu_pos
834834
)
835-
logging.info(
836-
"Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
837-
run_image, db_cpuset_cpus, command_str
838-
)
839-
)
840-
volumes = {}
841-
working_dir = "/"
842-
if mnt_point != "":
843-
volumes = {
844-
temporary_dir: {
845-
"bind": mnt_point,
846-
"mode": "rw",
847-
},
848-
}
849-
logging.info(
850-
f"setting volume as follow: {volumes}. working_dir={mnt_point}"
851-
)
852-
working_dir = mnt_point
853-
redis_container = docker_client.containers.run(
854-
image=run_image,
855-
volumes=volumes,
856-
auto_remove=True,
857-
privileged=True,
858-
working_dir=mnt_point,
859-
command=command_str,
860-
network_mode="host",
861-
detach=True,
862-
cpuset_cpus=db_cpuset_cpus,
863-
pid_mode="host",
864-
publish_all_ports=True,
835+
redis_container = start_redis_container(
836+
command_str,
837+
db_cpuset_cpus,
838+
docker_client,
839+
mnt_point,
840+
redis_containers,
841+
run_image,
842+
temporary_dir,
865843
)
866844

867-
time.sleep(5)
868-
869-
redis_containers.append(redis_container)
870-
871845
r = redis.StrictRedis(port=redis_proc_start_port)
872846
r.ping()
873847
redis_conns = [r]
@@ -1254,6 +1228,7 @@ def process_self_contained_coordinator_stream(
12541228
stdout=True, stderr=True
12551229
)
12561230
)
1231+
redis_container.remove()
12571232
except docker.errors.NotFound:
12581233
logging.info(
12591234
"When trying to fetch logs from DB container with id {} and image {} it was already stopped".format(
@@ -1272,6 +1247,7 @@ def process_self_contained_coordinator_stream(
12721247
for redis_container in redis_containers:
12731248
try:
12741249
redis_container.stop()
1250+
redis_container.remove()
12751251
except docker.errors.NotFound:
12761252
logging.info(
12771253
"When trying to stop DB container with id {} and image {} it was already stopped".format(
@@ -1285,6 +1261,7 @@ def process_self_contained_coordinator_stream(
12851261
if type(redis_container) == Container:
12861262
try:
12871263
redis_container.stop()
1264+
redis_container.remove()
12881265
except docker.errors.NotFound:
12891266
logging.info(
12901267
"When trying to stop Client container with id {} and image {} it was already stopped".format(
@@ -1479,6 +1456,50 @@ def process_self_contained_coordinator_stream(
14791456
return stream_id, overall_result, total_test_suite_runs
14801457

14811458

1459+
def start_redis_container(
1460+
command_str,
1461+
db_cpuset_cpus,
1462+
docker_client,
1463+
mnt_point,
1464+
redis_containers,
1465+
run_image,
1466+
temporary_dir,
1467+
auto_remove=False,
1468+
):
1469+
logging.info(
1470+
"Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
1471+
run_image, db_cpuset_cpus, command_str
1472+
)
1473+
)
1474+
volumes = {}
1475+
working_dir = "/"
1476+
if mnt_point != "":
1477+
volumes = {
1478+
temporary_dir: {
1479+
"bind": mnt_point,
1480+
"mode": "rw",
1481+
},
1482+
}
1483+
logging.info(f"setting volume as follow: {volumes}. working_dir={mnt_point}")
1484+
working_dir = mnt_point
1485+
redis_container = docker_client.containers.run(
1486+
image=run_image,
1487+
volumes=volumes,
1488+
auto_remove=auto_remove,
1489+
privileged=True,
1490+
working_dir=mnt_point,
1491+
command=command_str,
1492+
network_mode="host",
1493+
detach=True,
1494+
cpuset_cpus=db_cpuset_cpus,
1495+
pid_mode="host",
1496+
publish_all_ports=True,
1497+
)
1498+
time.sleep(5)
1499+
redis_containers.append(redis_container)
1500+
return redis_container
1501+
1502+
14821503
def filter_test_files(
14831504
defaults_filename,
14841505
priority_lower_limit,

‎tox.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ deps =
1818
-r{toxinidir}/dev_requirements.txt
1919

2020

21-
passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH
21+
passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH,TST_BINARY_REDIS_DIR
2222

2323
stoponfail =
2424
True

‎utils/tests/test_self_contained_coordinator.py

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from redisbench_admin.utils.remote import get_overall_dashboard_keynames
99
from redisbench_admin.utils.utils import get_ts_metric_name
10+
import logging
1011

1112
from redis_benchmarks_specification.__common__.env import (
1213
STREAM_KEYNAME_NEW_BUILD_EVENTS,
@@ -18,6 +19,7 @@
1819
)
1920
from redis_benchmarks_specification.__self_contained_coordinator__.self_contained_coordinator import (
2021
self_contained_coordinator_blocking_read,
22+
start_redis_container,
2123
)
2224

2325
from redis_benchmarks_specification.__self_contained_coordinator__.runners import (
@@ -31,6 +33,11 @@
3133
from utils.tests.test_data.api_builder_common import flow_1_and_2_api_builder_checks
3234

3335

36+
from redis_benchmarks_specification.__self_contained_coordinator__.docker import (
37+
generate_standalone_redis_server_args,
38+
)
39+
40+
3441
def test_extract_client_cpu_limit():
3542
with open(
3643
"./utils/tests/test_data/test-suites/redis-benchmark-full-suite-1Mkeys-100B.yml",
@@ -263,3 +270,51 @@ def test_self_contained_coordinator_blocking_read():
263270

264271
except redis.exceptions.ConnectionError:
265272
pass
273+
274+
275+
def test_start_redis_container():
276+
temporary_dir = os.getenv("TST_BINARY_REDIS_DIR", "")
277+
if temporary_dir == "":
278+
return
279+
280+
mnt_point = "/mnt/redis/"
281+
executable = f"{mnt_point}redis-server"
282+
redis_proc_start_port = 6379
283+
current_cpu_pos = 0
284+
ceil_db_cpu_limit = 1
285+
redis_configuration_parameters = None
286+
redis_arguments = ""
287+
docker_client = docker.from_env()
288+
redis_containers = []
289+
290+
command = generate_standalone_redis_server_args(
291+
executable,
292+
redis_proc_start_port,
293+
mnt_point,
294+
redis_configuration_parameters,
295+
redis_arguments,
296+
)
297+
command_str = " ".join(command)
298+
db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
299+
ceil_db_cpu_limit, current_cpu_pos
300+
)
301+
run_image = "gcc:8.5"
302+
redis_container = start_redis_container(
303+
command_str,
304+
db_cpuset_cpus,
305+
docker_client,
306+
mnt_point,
307+
redis_containers,
308+
run_image,
309+
temporary_dir,
310+
)
311+
r = redis.StrictRedis(port=redis_proc_start_port)
312+
try:
313+
r.ping()
314+
except redis.exceptions.ConnectionError:
315+
# Access and print the logs
316+
logs = redis_container.logs().decode("utf-8")
317+
logging.error("Container failed. Here are the logs:")
318+
logging.error(logs)
319+
raise
320+
redis_container.remove()

‎utils/tests/test_self_contained_coordinator_memtier.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ def test_self_contained_coordinator_blocking_read():
204204
assert len(datasink_conn.smembers(running_platforms_setname)) == 1
205205
assert len(datasink_conn.smembers(testcases_setname)) == 1
206206
assert len(datasink_conn.smembers(project_branches_setname)) == 1
207-
assert len(datasink_conn.smembers(project_versions_setname)) == 0
207+
assert len(datasink_conn.smembers(project_versions_setname)) == 1
208208

209209
except redis.exceptions.ConnectionError:
210210
pass

0 commit comments

Comments
 (0)