Skip to content

Commit c9278c0

Browse files
filipecosta90filipecosta90
authored
CI Tests Integration (single oss instance, and oss-cluster ) (#114)
1 parent 8e4dc0c commit c9278c0

File tree

6 files changed

+303
-3
lines changed

6 files changed

+303
-3
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 91 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,102 @@ jobs:
66
build-ubuntu:
77
strategy:
88
matrix:
9-
platform: [ubuntu-latest, ubuntu-16.04]
9+
platform: [ubuntu-latest,ubuntu-16.04]
1010
runs-on: ${{ matrix.platform }}
1111
steps:
12-
- uses: actions/checkout@v1
12+
- uses: actions/checkout@v2
1313
- name: Install dependencies
14-
run: sudo apt-get install autoconf automake pkg-config libevent-dev libpcre3-dev libssl-dev
14+
run: |
15+
sudo apt-get -qq update
16+
sudo apt-get install autoconf automake pkg-config libevent-dev libpcre3-dev libssl-dev
1517
- name: Build
1618
run: autoreconf -ivf && ./configure && make
19+
- name: Setup Python
20+
uses: actions/setup-python@v1
21+
with:
22+
python-version: '3.6'
23+
24+
- name: Cache pip
25+
uses: actions/cache@v1
26+
with:
27+
path: ~/.cache/pip # This path is specific to Ubuntu
28+
# Look to see if there is a cache hit for the corresponding requirements file
29+
key: ${{ runner.os }}-pip-${{ hashFiles('tests/test_requirements.txt') }}
30+
restore-keys: |
31+
${{ runner.os }}-pip-
32+
${{ runner.os }}-
33+
- name: Install Python dependencies
34+
run: pip install -r tests/test_requirements.txt
35+
36+
- name: Cache Redis
37+
id: cache-redis
38+
uses: actions/cache@v1
39+
with:
40+
path: /home/runner/work/memtier_benchmark/memtier_benchmark/redis
41+
key: ${{ runner.os }}-redis
42+
43+
- name: Install Redis Server test dependencies
44+
if: steps.cache-redis.outputs.cache-hit != 'true'
45+
run: |
46+
git clone git://github.com/antirez/redis.git --branch unstable
47+
cd redis
48+
make BUILD_TLS=yes
49+
./utils/gen-test-certs.sh
50+
./src/redis-server --version
51+
cd ..
52+
53+
- name: Test OSS TCP
54+
run: |
55+
cd tests
56+
MEMTIER_BINARY=./../memtier_benchmark \
57+
python3 -m RLTest \
58+
--env oss -v --clear-logs \
59+
--oss-redis-path ../redis/src/redis-server
60+
cd ..
61+
62+
- name: Test OSS TCP TLS
63+
if: matrix.platform == 'ubuntu-latest'
64+
run: |
65+
cd tests
66+
TLS_CERT=../redis/tests/tls/redis.crt \
67+
TLS_KEY=../redis/tests/tls/redis.key \
68+
TLS_CACERT=../redis/tests/tls/ca.crt \
69+
MEMTIER_BINARY=../memtier_benchmark \
70+
python3 -m RLTest \
71+
--env oss -v --clear-logs \
72+
--oss-redis-path ../redis/src/redis-server \
73+
--tls-cert-file ../redis/tests/tls/redis.crt \
74+
--tls-key-file ../redis/tests/tls/redis.key \
75+
--tls-ca-cert-file ../redis/tests/tls/ca.crt \
76+
--tls
77+
cd ..
78+
79+
- name: Test OSS-CLUSTER TCP
80+
run: |
81+
cd tests
82+
MEMTIER_BINARY=./../memtier_benchmark \
83+
python3 -m RLTest \
84+
--env oss-cluster -v --clear-logs --shards-count 3 \
85+
--oss-redis-path ../redis/src/redis-server
86+
cd ..
87+
88+
- name: Test OSS-CLUSTER TCP TLS
89+
if: matrix.platform == 'ubuntu-latest'
90+
run: |
91+
cd tests
92+
TLS_CERT=../redis/tests/tls/redis.crt \
93+
TLS_KEY=../redis/tests/tls/redis.key \
94+
TLS_CACERT=../redis/tests/tls/ca.crt \
95+
MEMTIER_BINARY=../memtier_benchmark \
96+
python3 -m RLTest \
97+
--env oss-cluster --shards-count 3 -v --clear-logs \
98+
--oss-redis-path ../redis/src/redis-server \
99+
--tls-cert-file ../redis/tests/tls/redis.crt \
100+
--tls-key-file ../redis/tests/tls/redis.key \
101+
--tls-ca-cert-file ../redis/tests/tls/ca.crt \
102+
--tls
103+
cd ..
104+
17105
build-macos:
18106
strategy:
19107
matrix:

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ config.log
1919
config.status
2020
libtool
2121
stamp-h1
22+
.idea
23+
.vscode
24+
__pycache__

‎tests/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
logs

‎tests/include.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import glob
2+
import os
3+
4+
MEMTIER_BINARY = os.environ.get("MEMTIER_BINARY", "memtier_benchmark")
5+
TLS_CERT = os.environ.get("TLS_CERT", "")
6+
TLS_KEY = os.environ.get("TLS_KEY", "")
7+
TLS_CACERT = os.environ.get("TLS_CACERT", "")
8+
9+
10+
def assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
11+
overall_request_count):
12+
# assert correct exit code
13+
env.assertTrue(memtier_ok == True)
14+
# assert we have all outputs
15+
env.assertTrue(os.path.isfile('{0}/mb.stdout'.format(config.results_dir)))
16+
env.assertTrue(os.path.isfile('{0}/mb.stderr'.format(config.results_dir)))
17+
env.assertTrue(os.path.isfile('{0}/mb.json'.format(config.results_dir)))
18+
19+
# assert we have the expected request count
20+
env.assertEqual(overall_expected_request_count, overall_request_count)
21+
22+
23+
def add_required_env_arguments(benchmark_specs, config, env, master_nodes_list):
24+
# check if environment is cluster
25+
if env.isCluster():
26+
benchmark_specs["args"].append("--cluster-mode")
27+
# check if environment uses Unix Socket connections
28+
if env.isUnixSocket():
29+
benchmark_specs["args"].append("--unix-socket")
30+
benchmark_specs["args"].append(master_nodes_list[0]['unix_socket_path'])
31+
config["memtier_benchmark"]['explicit_connect_args'] = True
32+
else:
33+
config['redis_process_port'] = master_nodes_list[0]['port']
34+
35+
36+
def debugPrintMemtierOnError(config, env, memtier_ok):
37+
if not memtier_ok:
38+
with open('{0}/mb.stderr'.format(config.results_dir)) as stderr:
39+
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
40+
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stderr'.format(config.results_dir)), True)
41+
for line in stderr:
42+
env.debugPrint(line.rstrip(), True)
43+
44+
with open('{0}/mb.stdout'.format(config.results_dir)) as stderr:
45+
env.debugPrint("### PRINTING STDERR OUTPUT OF MEMTIER ON FAILURE ###", True)
46+
env.debugPrint("### mb.stderr file location: {0}".format('{0}/mb.stdout'.format(config.results_dir)), True)
47+
for line in stderr:
48+
env.debugPrint(line.rstrip(), True)
49+
50+
if not env.isCluster():
51+
if env.envRunner is not None:
52+
log_file = os.path.join(env.envRunner.dbDirPath, env.envRunner._getFileName('master', '.log'))
53+
with open(log_file) as redislog:
54+
env.debugPrint("### REDIS LOG ###", True)
55+
env.debugPrint(
56+
"### log_file file location: {0}".format(log_file), True)
57+
for line in redislog:
58+
env.debugPrint(line.rstrip(), True)
59+
60+
61+
def get_expected_request_count(config):
62+
result = -1
63+
if 'memtier_benchmark' in config:
64+
mt = config['memtier_benchmark']
65+
if 'threads' in mt and 'clients' in mt and 'requests' in mt:
66+
result = config['memtier_benchmark']['threads'] * config['memtier_benchmark']['clients'] * \
67+
config['memtier_benchmark']['requests']
68+
return result
69+
70+
71+
def agg_info_commandstats(master_nodes_connections, merged_command_stats):
72+
overall_request_count = 0
73+
for master_connection in master_nodes_connections:
74+
shard_stats = master_connection.execute_command("INFO", "COMMANDSTATS")
75+
for cmd_name, cmd_stat in shard_stats.items():
76+
if cmd_name in merged_command_stats:
77+
overall_request_count += cmd_stat['calls']
78+
merged_command_stats[cmd_name]['calls'] = merged_command_stats[cmd_name]['calls'] + cmd_stat['calls']
79+
return overall_request_count
80+
81+
82+
def addTLSArgs(benchmark_specs, env):
83+
if env.useTLS:
84+
benchmark_specs['args'].append('--tls')
85+
benchmark_specs['args'].append('--cert={}'.format(TLS_CERT))
86+
benchmark_specs['args'].append('--key={}'.format(TLS_KEY))
87+
benchmark_specs['args'].append('--cacert={}'.format(TLS_CACERT))
88+
89+
90+
def get_default_memtier_config():
91+
config = {
92+
"memtier_benchmark": {
93+
"binary": MEMTIER_BINARY,
94+
"threads": 10,
95+
"clients": 5,
96+
"requests": 1000
97+
},
98+
}
99+
return config
100+
101+
102+
def ensure_clean_benchmark_folder(dirname):
103+
files = glob.glob('{}/*'.format(dirname))
104+
for f in files:
105+
os.remove(f)
106+
if os.path.exists(dirname):
107+
os.removedirs(dirname)
108+
os.makedirs(dirname)

‎tests/test_requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
redis>=3.0.0
2+
git+https://github.com/Grokzen/redis-py-cluster.git@master
3+
git+https://github.com/RedisLabsModules/RLTest.git@master
4+
git+https://github.com/RedisLabs/mbdirector.git@master

‎tests/tests_oss_simple_flow.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import tempfile
2+
3+
from include import *
4+
from mbdirector.benchmark import Benchmark
5+
from mbdirector.runner import RunConfig
6+
7+
8+
def test_default_set_get(env):
9+
benchmark_specs = {"name": env.testName, "args": []}
10+
addTLSArgs(benchmark_specs, env)
11+
config = get_default_memtier_config()
12+
master_nodes_list = env.getMasterNodesList()
13+
overall_expected_request_count = get_expected_request_count(config)
14+
15+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
16+
17+
# Create a temporary directory
18+
test_dir = tempfile.mkdtemp()
19+
20+
config = RunConfig(test_dir, env.testName, config, {})
21+
ensure_clean_benchmark_folder(config.results_dir)
22+
23+
benchmark = Benchmark.from_json(config, benchmark_specs)
24+
25+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
26+
memtier_ok = benchmark.run()
27+
debugPrintMemtierOnError(config, env, memtier_ok)
28+
29+
master_nodes_connections = env.getOSSMasterNodesConnectionList()
30+
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
31+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
32+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
33+
overall_request_count)
34+
35+
36+
# run each test on different env
37+
def test_default_set_get_1_1(env):
38+
benchmark_specs = {"name": env.testName, "args": ['--ratio=1:1']}
39+
addTLSArgs(benchmark_specs, env)
40+
config = get_default_memtier_config()
41+
master_nodes_list = env.getMasterNodesList()
42+
overall_expected_request_count = get_expected_request_count(config)
43+
44+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
45+
46+
# Create a temporary directory
47+
test_dir = tempfile.mkdtemp()
48+
49+
config = RunConfig(test_dir, env.testName, config, {})
50+
ensure_clean_benchmark_folder(config.results_dir)
51+
52+
benchmark = Benchmark.from_json(config, benchmark_specs)
53+
54+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
55+
memtier_ok = benchmark.run()
56+
debugPrintMemtierOnError(config, env, memtier_ok)
57+
58+
master_nodes_connections = env.getOSSMasterNodesConnectionList()
59+
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
60+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
61+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
62+
overall_request_count)
63+
64+
# assert same number of gets and sets
65+
env.assertEqual(merged_command_stats['cmdstat_set']['calls'], merged_command_stats['cmdstat_get']['calls'])
66+
67+
68+
# run each test on different env
69+
def test_default_set_get_3_runs(env):
70+
env.skipOnCluster()
71+
run_count = 3
72+
benchmark_specs = {"name": env.testName, "args": ['--run-count={}'.format(run_count)]}
73+
addTLSArgs(benchmark_specs, env)
74+
config = get_default_memtier_config()
75+
master_nodes_list = env.getMasterNodesList()
76+
overall_expected_request_count = get_expected_request_count(config) * run_count
77+
78+
add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
79+
80+
# Create a temporary directory
81+
test_dir = tempfile.mkdtemp()
82+
83+
config = RunConfig(test_dir, env.testName, config, {})
84+
ensure_clean_benchmark_folder(config.results_dir)
85+
86+
benchmark = Benchmark.from_json(config, benchmark_specs)
87+
88+
# benchmark.run() returns True if the return code of memtier_benchmark was 0
89+
memtier_ok = benchmark.run()
90+
debugPrintMemtierOnError(config, env, memtier_ok)
91+
92+
master_nodes_connections = env.getOSSMasterNodesConnectionList()
93+
merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
94+
overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
95+
assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count,
96+
overall_request_count)

0 commit comments

Comments
 (0)