Skip to content

Commit 823c31f

Browse files
filipecosta90filipecosta90
authored
Reduced the ammount of collected data on server stats and commandstats (#277)
1 parent 7e198ba commit 823c31f

File tree

5 files changed

+80
-35
lines changed

5 files changed

+80
-35
lines changed

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redisbench-admin"
3-
version = "0.6.14"
3+
version = "0.6.15"
44
description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
55
authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
66
readme = "README.md"

‎redisbench_admin/run/args.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,12 @@ def common_run_args(parser):
127127
action="store_true",
128128
help="uploads the results to RedisTimeSeries. Proper credentials are required",
129129
)
130+
parser.add_argument(
131+
"--collect_commandstats",
132+
default=False,
133+
action="store_true",
134+
help="Enables commandstats collection.",
135+
)
130136
parser.add_argument(
131137
"--allowed-envs",
132138
type=str,

‎redisbench_admin/run/grafana.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,16 @@
44
# All rights reserved.
55
#
66
import logging
7+
import time
78

89
import pytablewriter
910
from pytablewriter import MarkdownTableWriter
1011

12+
# 7 days expire
13+
STALL_INFO_DAYS = 7
14+
EXPIRE_TIME_SECS_PROFILE_KEYS = 60 * 60 * 24 * STALL_INFO_DAYS
15+
EXPIRE_TIME_MSECS_PROFILE_KEYS = EXPIRE_TIME_SECS_PROFILE_KEYS * 1000
16+
1117

1218
def generate_artifacts_table_grafana_redis(
1319
args,
@@ -98,6 +104,8 @@ def generate_artifacts_table_grafana_redis(
98104
profile_id,
99105
)
100106
if args.push_results_redistimeseries:
107+
current_time = time.time() * 1000
108+
timeframe_by_branch = current_time - EXPIRE_TIME_MSECS_PROFILE_KEYS
101109
rts.redis.zadd(
102110
zset_profiles_setups_testcases_branches,
103111
{tf_github_branch: start_time_ms},
@@ -122,9 +130,21 @@ def generate_artifacts_table_grafana_redis(
122130
zset_profiles,
123131
{profile_id: start_time_ms},
124132
)
133+
sorted_set_keys = [
134+
zset_profiles,
135+
zset_profiles_setups_testcases_profileid,
136+
zset_profiles_setups_testcases,
137+
zset_profiles_setup,
138+
zset_profiles_setups_testcases_branches_latest_link,
139+
]
140+
for keyname in sorted_set_keys:
141+
rts.redis.zremrangebyscore(keyname, 0, int(timeframe_by_branch))
142+
125143
rts.redis.sadd(profile_set_redis_key, test_name)
126-
rts.redis.set(
144+
rts.redis.expire(profile_set_redis_key, EXPIRE_TIME_SECS_PROFILE_KEYS)
145+
rts.redis.setex(
127146
profile_string_testcase_markdown_key,
147+
EXPIRE_TIME_SECS_PROFILE_KEYS,
128148
profile_markdown_str,
129149
)
130150
logging.info(

‎redisbench_admin/run/metrics.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,9 @@ def extract_results_table(
8484
return results_matrix
8585

8686

87-
def collect_redis_metrics(redis_conns, sections=["memory", "cpu", "commandstats"]):
87+
def collect_redis_metrics(
88+
redis_conns, sections=["memory", "cpu", "commandstats"], section_filter=None
89+
):
8890
start_time = dt.datetime.utcnow()
8991
start_time_ms = int((start_time - dt.datetime(1970, 1, 1)).total_seconds() * 1000)
9092
res = []
@@ -100,11 +102,16 @@ def collect_redis_metrics(redis_conns, sections=["memory", "cpu", "commandstats"
100102
if section not in overall:
101103
overall[section] = {}
102104
for k, v in info.items():
103-
if type(v) is float or type(v) is int:
105+
collect = True
106+
if section_filter is not None:
107+
if section in section_filter:
108+
if k not in section_filter[section]:
109+
collect = False
110+
if collect and type(v) is float or type(v) is int:
104111
if k not in overall[section]:
105112
overall[section][k] = 0
106113
overall[section][k] += v
107-
if type(v) is dict:
114+
if collect and type(v) is dict:
108115
for inner_k, inner_v in v.items():
109116
if type(inner_v) is float or type(inner_v) is int:
110117
final_str_k = "{}_{}".format(k, inner_k)

‎redisbench_admin/run_remote/run_remote.py‎

Lines changed: 42 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@
6363
)
6464

6565

66+
# 7 days expire
67+
STALL_INFO_DAYS = 7
68+
EXPIRE_TIME_SECS_PROFILE_KEYS = 60 * 60 * 24 * STALL_INFO_DAYS
69+
EXPIRE_TIME_MSECS_PROFILE_KEYS = EXPIRE_TIME_SECS_PROFILE_KEYS * 1000
70+
71+
6672
def run_remote_command_logic(args, project_name, project_version):
6773
logging.info(
6874
"Using: {project_name} {project_version}".format(
@@ -383,12 +389,6 @@ def run_remote_command_logic(args, project_name, project_version):
383389
db_ssh_port,
384390
)
385391

386-
(
387-
_,
388-
_,
389-
overall_start_time_metrics,
390-
) = collect_redis_metrics(redis_conns)
391-
392392
(
393393
start_time,
394394
start_time_ms,
@@ -539,7 +539,14 @@ def run_remote_command_logic(args, project_name, project_version):
539539
_,
540540
overall_end_time_metrics,
541541
) = collect_redis_metrics(
542-
redis_conns, ["memory"]
542+
redis_conns,
543+
["memory"],
544+
{
545+
"memory": [
546+
"used_memory",
547+
"used_memory_dataset",
548+
]
549+
},
543550
)
544551
expire_ms = 7 * 24 * 60 * 60 * 1000
545552
export_redis_metrics(
@@ -557,28 +564,29 @@ def run_remote_command_logic(args, project_name, project_version):
557564
{"metric-type": "redis-metrics"},
558565
expire_ms,
559566
)
560-
(
561-
end_time_ms,
562-
_,
563-
overall_commandstats_metrics,
564-
) = collect_redis_metrics(
565-
redis_conns, ["commandstats"]
566-
)
567-
export_redis_metrics(
568-
artifact_version,
569-
end_time_ms,
570-
overall_commandstats_metrics,
571-
rts,
572-
setup_name,
573-
setup_type,
574-
test_name,
575-
tf_github_branch,
576-
tf_github_org,
577-
tf_github_repo,
578-
tf_triggering_env,
579-
{"metric-type": "commandstats"},
580-
expire_ms,
581-
)
567+
if args.collect_commandstats:
568+
(
569+
end_time_ms,
570+
_,
571+
overall_commandstats_metrics,
572+
) = collect_redis_metrics(
573+
redis_conns, ["commandstats"]
574+
)
575+
export_redis_metrics(
576+
artifact_version,
577+
end_time_ms,
578+
overall_commandstats_metrics,
579+
rts,
580+
setup_name,
581+
setup_type,
582+
test_name,
583+
tf_github_branch,
584+
tf_github_org,
585+
tf_github_repo,
586+
tf_triggering_env,
587+
{"metric-type": "commandstats"},
588+
expire_ms,
589+
)
582590

583591
if setup_details["env"] is None:
584592
if args.keep_env_and_topo is False:
@@ -781,7 +789,11 @@ def run_remote_command_logic(args, project_name, project_version):
781789
)
782790
profile_markdown_str = htmlwriter.dumps()
783791
profile_markdown_str = profile_markdown_str.replace("\n", "")
784-
rts.redis.set(target_tables_latest_key, profile_markdown_str)
792+
rts.redis.setex(
793+
target_tables_latest_key,
794+
EXPIRE_TIME_SECS_PROFILE_KEYS,
795+
profile_markdown_str,
796+
)
785797
exit(return_code)
786798

787799

0 commit comments

Comments
 (0)