Skip to content

Commit 642d560

Browse files
chenziliangchenziliang
committed
perf cases refinement
1 parent c0b3986 commit 642d560

File tree

3 files changed

+94
-75
lines changed

3 files changed

+94
-75
lines changed

‎ci/perf.py‎

Lines changed: 71 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -5,93 +5,93 @@
55

66

77
nozzle_perf_params = [
8-
{"hec-workers": "1", "hec-batch-size": "100"},
9-
{"hec-workers": "2", "hec-batch-size": "100"},
10-
{"hec-workers": "4", "hec-batch-size": "100"},
11-
{"hec-workers": "8", "hec-batch-size": "100"},
12-
{"hec-workers": "16", "hec-batch-size": "100"},
13-
{"hec-workers": "32", "hec-batch-size": "100"},
14-
{"hec-workers": "1", "hec-batch-size": "1000"},
15-
{"hec-workers": "2", "hec-batch-size": "1000"},
16-
{"hec-workers": "4", "hec-batch-size": "1000"},
17-
{"hec-workers": "8", "hec-batch-size": "1000"},
18-
{"hec-workers": "16", "hec-batch-size": "1000"},
19-
{"hec-workers": "32", "hec-batch-size": "1000"},
8+
{'hec-workers': '1', 'hec-batch-size': '100'},
9+
{'hec-workers': '2', 'hec-batch-size': '100'},
10+
{'hec-workers': '4', 'hec-batch-size': '100'},
11+
{'hec-workers': '8', 'hec-batch-size': '100'},
12+
{'hec-workers': '16', 'hec-batch-size': '100'},
13+
{'hec-workers': '32', 'hec-batch-size': '100'},
14+
{'hec-workers': '1', 'hec-batch-size': '1000'},
15+
{'hec-workers': '2', 'hec-batch-size': '1000'},
16+
{'hec-workers': '4', 'hec-batch-size': '1000'},
17+
{'hec-workers': '8', 'hec-batch-size': '1000'},
18+
{'hec-workers': '16', 'hec-batch-size': '1000'},
19+
{'hec-workers': '32', 'hec-batch-size': '1000'},
2020
]
2121

2222
nozzle_perf_suites = [
2323
{
24-
"message-type": "s256byte",
25-
"extra-fields": "message_type:s256byte",
26-
"cases": nozzle_perf_params,
24+
'message-type': 's256byte',
25+
'extra-fields': 'message_type:s256byte',
26+
'cases': nozzle_perf_params,
2727
},
2828
{
29-
"message-type": "s1kbyte",
30-
"extra-fields": "message_type:s1kbyte",
31-
"cases": nozzle_perf_params,
29+
'message-type': 's1kbyte',
30+
'extra-fields': 'message_type:s1kbyte',
31+
'cases': nozzle_perf_params,
3232
},
3333
{
34-
"message-type": "uns256byte",
35-
"extra-fields": "message_type:uns256byte",
36-
"cases": nozzle_perf_params,
34+
'message-type': 'uns256byte',
35+
'extra-fields': 'message_type:uns256byte',
36+
'cases': nozzle_perf_params,
3737
},
3838
{
39-
"message-type": "uns1kbyte",
40-
"extra-fields": "message_type:uns1kbyte",
41-
"cases": nozzle_perf_params,
39+
'message-type': 'uns1kbyte',
40+
'extra-fields': 'message_type:uns1kbyte',
41+
'cases': nozzle_perf_params,
4242
},
4343
]
4444

4545

4646
def run_nozzle_perf(config):
4747
for suite in nozzle_perf_suites:
48-
for case in suite["cases"]:
49-
kvs = ",".join("{}:{}".format(k, v) for k, v in case.iteritems())
50-
extra_fields = "{},{}".format(suite["extra-fields"], kvs)
48+
for case in suite['cases']:
49+
kvs = ','.join('{}:{}'.format(k, v) for k, v in case.iteritems())
50+
extra_fields = '{},{}'.format(suite['extra-fields'], kvs)
5151

5252
cmd = [
53-
"./splunk-firehose-nozzle",
54-
"--api-endpoint", config["api-endpoint"],
55-
"--user", config["user"],
56-
"--password", config["password"],
57-
"--splunk-host", config["splunk-host"],
58-
"--splunk-token", config["splunk-token"],
59-
"--splunk-index", config["splunk-index"],
60-
"--hec-workers", case["hec-workers"],
61-
"--hec-batch-size", case["hec-batch-size"],
62-
"--events", "ContainerMetric,CounterEvent,Error,HttpStart,HttpStartStop,HttpStop,LogMessage,ValueMetric",
63-
"--extra-fields", extra_fields,
64-
"--add-app-info",
65-
"--enable-event-tracing",
66-
"--skip-ssl-validation-cf",
67-
"--skip-ssl-validation-splunk",
53+
'./splunk-firehose-nozzle',
54+
'--api-endpoint', config['api-endpoint'],
55+
'--user', config['user'],
56+
'--password', config['password'],
57+
'--splunk-host', config['splunk-host'],
58+
'--splunk-token', config['splunk-token'],
59+
'--splunk-index', config['splunk-index'],
60+
'--hec-workers', case['hec-workers'],
61+
'--hec-batch-size', case['hec-batch-size'],
62+
'--events', 'ContainerMetric,CounterEvent,Error,HttpStart,HttpStartStop,HttpStop,LogMessage,ValueMetric',
63+
'--extra-fields', extra_fields,
64+
'--add-app-info',
65+
'--enable-event-tracing',
66+
'--skip-ssl-validation-cf',
67+
'--skip-ssl-validation-splunk',
6868
]
69-
print " ".join(cmd)
70-
execute(cmd, config["duration"])
69+
print ' '.join(cmd)
70+
execute(cmd, config['duration'])
7171

7272

7373
def execute(cmd, duration):
7474
has_error = False
7575
while 1:
7676
try:
7777
start = time.time()
78-
print "start {} {}".format(" ".join(cmd), start)
78+
print 'start {} {}'.format(' '.join(cmd), start)
7979
out, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
8080
except Exception as e:
8181
has_error = True
8282
print e
8383
else:
8484
has_error = False
85-
print "out:", out
86-
print "err:", err
85+
print 'out:', out
86+
print 'err:', err
8787
finally:
8888
end = time.time()
8989
last = end - start
90-
print "end {} {} duration={}".format(" ".join(cmd), end, last)
90+
print 'end {} {} duration={}'.format(' '.join(cmd), end, last)
9191

9292
# If we run too short, rerun it
9393
if last < 0.8 * duration:
94-
print "run too short, retry..."
94+
print 'run too short, retry...'
9595
time.sleep(1)
9696
elif has_error:
9797
pass
@@ -101,43 +101,43 @@ def execute(cmd, duration):
101101

102102
def run_trafficcontroller(duration):
103103
for suite in nozzle_perf_suites:
104-
for case in suite["cases"]:
105-
ip = "pcf_{}_{}".format(
106-
suite["message-type"],
107-
"_".join("{}-{}".format(k, v) for k, v in case.iteritems()))
104+
for case in suite['cases']:
105+
ip = 'pcf_{}_{}'.format(
106+
suite['message-type'],
107+
'_'.join('{}-{}'.format(k, v) for k, v in case.iteritems()))
108108
cmd = [
109-
"./trafficcontroller",
110-
"--config", "loggregator_trafficcontroller.json",
111-
"--disableAccessControl",
112-
"--duration", str(duration),
113-
"--message-type", suite["message-type"],
114-
"--ip", ip,
109+
'./trafficcontroller',
110+
'--config', 'loggregator_trafficcontroller.json',
111+
'--disableAccessControl',
112+
'--duration', str(duration),
113+
'--message-type', suite['message-type'],
114+
'--ip', ip,
115115
]
116116

117117
execute(cmd, duration)
118118
time.sleep(10)
119119

120120

121121
def main():
122-
parser = argparse.ArgumentParser(description="Nozzle perf test driver")
123-
parser.add_argument("--run", dest="run", required=True, help="nozzle or trafficcontroller")
124-
parser.add_argument("--duration", dest="duration", type=int, required=True, help="how long to run in seconds")
122+
parser = argparse.ArgumentParser(description='Nozzle perf test driver')
123+
parser.add_argument('--run', dest='run', required=True, help='nozzle or trafficcontroller')
124+
parser.add_argument('--duration', dest='duration', type=int, required=True, help='how long to run in seconds')
125125
args = parser.parse_args()
126126

127-
if args.run == "nozzle":
127+
if args.run == 'nozzle':
128128
config = {
129-
"api-endpoint": os.environ.get("API_ENDPOINT", "http://trafficcontroller:9911"),
130-
"user": os.environ.get("API_USER", "admin"),
131-
"password": os.environ.get("API_PASSWORD", "admin"),
132-
"splunk-host": os.environ.get("SPLUNK_HOST", "https://heclb1:8088"),
133-
"splunk-token": os.environ.get("SPLUNK_TOKEN", "00000000-0000-0000-0000-000000000000"),
134-
"splunk-index": os.environ.get("SPLUNK_INDEX", "main"),
135-
"duration": args.duration,
129+
'api-endpoint': os.environ.get('API_ENDPOINT', 'http://trafficcontroller:9911'),
130+
'user': os.environ.get('API_USER', 'admin'),
131+
'password': os.environ.get('API_PASSWORD', 'admin'),
132+
'splunk-host': os.environ.get('SPLUNK_HOST', 'https://heclb1:8088'),
133+
'splunk-token': os.environ.get('SPLUNK_TOKEN', '00000000-0000-0000-0000-000000000000'),
134+
'splunk-index': os.environ.get('SPLUNK_INDEX', 'main'),
135+
'duration': args.duration,
136136
}
137137
run_nozzle_perf(config)
138138
else:
139139
run_trafficcontroller(args.duration)
140140

141141

142-
if __name__ == "__main__":
142+
if __name__ == '__main__':
143143
main()

‎ci/services.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
trafficcontroller:
2-
image: zlchen/trafficcontroller:1.1
2+
image: zlchen/trafficcontroller:latest
33
hostname: trafficcontroller
44
ports:
55
- "8081"
66
- "9911"
77
restart: always
88

99
nozzle:
10-
image: zlchen/nozzle:1.1
10+
image: zlchen/nozzle:latest
1111
hostname: nozzle
1212
ports:
1313
- "9912"
@@ -20,6 +20,4 @@ nozzle:
2020
- SPLUNK_INDEX=main
2121
- SKIP_SSL_VALIDATION_CF=true
2222
- SKIP_SSL_VALIDATION_SPLUNK=true
23-
depends_on:
24-
- trafficcontroller
2523
restart: always

‎ci/trafficcontroller.dockerfile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM golang:1.8.3
2+
3+
RUN apt-get update && \
4+
apt-get install -y telnet && \
5+
apt-get install -y python
6+
7+
8+
ENV GOPATH /go
9+
RUN go get -d github.com/chenziliang/loggregator; exit 0
10+
WORKDIR /go/src/github.com/chenziliang/loggregator
11+
RUN git checkout feature/firehose-standalone
12+
ENV GOPATH /go/src/github.com/chenziliang/loggregator
13+
RUN cd /go/src/github.com/chenziliang/loggregator && ./scripts/build
14+
WORKDIR /go/src/github.com/chenziliang/loggregator
15+
RUN mv ./bin/trafficcontroller .
16+
RUN git clone https://github.com/cloudfoundry-community/splunk-firehose-nozzle
17+
18+
EXPOSE 9911
19+
EXPOSE 8081
20+
21+
CMD ["/usr/bin/python", "splunk-firehose-nozzle/ci/perf.py", "--run", "trafficcontroller", "--duration", "1200"]

0 commit comments

Comments
 (0)