Skip to content

Commit ec034c4

Browse files
chenziliangchenziliang
authored
Merge pull request #112 from cloudfoundry-community/develop
Develop
2 parents bc5cfdb + 0f8f0bb commit ec034c4

File tree

9 files changed

+174
-90
lines changed

9 files changed

+174
-90
lines changed

‎.gitignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
.idea
22

3-
nozzle.sh
4-
53
tools/dump_app_info/dump_app_info
64
tools/data_gen/data_gen
75

6+
87
cache.db
98
*.out
109

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ In addition, logs from the nozzle itself are of sourcetype `cf:splunknozzle`.
1919
### Setup
2020

2121
The Nozzle requires a user with the scope `doppler.firehose` and
22-
`cloud_controller.admin_read_only` (the latter is only required if `ADD_APP_INFO` is true). If `cloud_controller.admin` is not
22+
`cloud_controller.admin_read_only` (the latter is only required if `ADD_APP_INFO` is true). If `cloud_controller.admin_read_only` is not
2323
available in the system, switch to use `cloud_controller.admin`.
2424

2525
You can either
@@ -49,7 +49,7 @@ or later. Earlier versions should use `cloud_controller.admin` instead.
4949

5050
- - - -
5151
#### Environment Parameters
52-
You can declare parameters by making a copy of the scripts/nozzle.sh.template.
52+
You can declare parameters by making a copy of the scripts/nozzle.sh.template.
5353
* `DEBUG`: Enable debug mode (forward to standard out instead of Splunk).
5454

5555
__Cloud Foundry configuration parameters:__
@@ -68,7 +68,7 @@ __Advanced Configuration Features:__
6868
* `JOB_HOST`: Tags nozzle log events with job host.
6969
* `SKIP_SSL_VALIDATION_CF`: Skips SSL certificate validation for connection to Cloud Foundry. Secure communications will not check SSL certificates against a trusted certificate authority.
7070
This is recommended for dev environments only.
71-
* `SKIP_SSL_VALIDATION_SPLUNK`: Skips SSL certificate validation for connection to Splunk. Secure communications will not check SSL certificates against a trusted certificate authority.
71+
* `SKIP_SSL_VALIDATION_SPLUNK`: Skips SSL certificate validation for connection to Splunk. Secure communications will not check SSL certificates against a trusted certificate authority.
7272
This is recommended for dev environments only.
7373
* `FIREHOSE_SUBSCRIPTION_ID`: Tags nozzle events with a Firehose subscription id. See https://docs.pivotal.io/pivotalcf/1-11/loggregator/log-ops-guide.html.
7474
* `FIREHOSE_KEEP_ALIVE`: Keep alive duration for the Firehose consumer.

‎ci/Jenkinsfile‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,9 @@ withSplunkWrapNode("master") {
2424
splunkRunScript request:dockerReq,
2525
script:
2626
"""
27-
pip install splunk_orca==0.7.5 -i https://repo.splunk.com/artifactory/api/pypi/pypi-virtual/simple
27+
pip install splunk_orca==0.8.0 -i https://repo.splunk.com/artifactory/api/pypi/pypi-virtual/simple
2828
cd ci
2929
30-
# Workaround to use 0.8.0
31-
# sed -i"" "s/^splunk_image_version = 0.7/splunk_image_version = 0.8/g" /build/.orca/orca.conf
32-
#sed -i"" "s/^splunk_ansible_version/#splunk_ansible_version/g" /build/.orca/orca.conf
3330
splunk_orca create --sc nozzle
3431
""";
3532
}

‎ci/nozzle.dockerfile‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
FROM golang:1.8.3
2+
3+
RUN apt-get update && \
4+
apt-get install -y telnet && \
5+
apt-get install -y gcc && \
6+
apt-get install -y screen && \
7+
apt-get install -y python-dev && \
8+
apt-get install -y python-pip && \
9+
apt-get install -y python
10+
11+
RUN pip install psutil && pip install requests --upgrade
12+
13+
ENV GOPATH /go
14+
15+
ENV HEC_WORKERS=8
16+
ENV ADD_APP_INFO=true
17+
ENV SKIP_SSL_VALIDATION_CF=true
18+
ENV SKIP_SSL_VALIDATION_SPLUNK=true
19+
20+
ENV API_ENDPOINT=http://trafficcontroller:9911
21+
ENV API_USER=admin
22+
ENV API_PASSWORD=admin
23+
24+
ENV EVENTS=ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
25+
ENV SPLUNK_TOKEN=1CB57F19-DC23-419A-8EDA-BA545DD3674D
26+
ENV SPLUNK_HOST=https://heclb1:8088
27+
ENV SPLUNK_INDEX=main
28+
ENV FLUSH_INTERVAL=30s
29+
ENV FIREHOSE_SUBSCRIPTION_ID=spl-nozzle-perf-testing
30+
31+
ENV NOZZLE_BRANCH=master
32+
33+
ADD nozzle.sh /
34+
35+
CMD ["/bin/sh", "-c", "/nozzle.sh"]

‎ci/nozzle.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/sh
2+
3+
# Workaround to make sure dependency service is up and running
4+
sleep 60
5+
6+
branch=${NOZZLE_BRANCH:-master}
7+
8+
# Build nozzle
9+
go get -d github.com/cloudfoundry-community/splunk-firehose-nozzle
10+
cd /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
11+
git checkout ${branch}
12+
make build
13+
14+
15+
# Run proc_monitor
16+
echo "Run proc monitor"
17+
git clone https://github.com/chenziliang/proc_monitor && git checkout develop
18+
cd proc_monitor
19+
screen -S proc_monitor -m -d python proc_monitor.py
20+
cd ..
21+
22+
duration=${NOZZLE_DURATION:-1200}
23+
24+
# Run perf test
25+
echo "Run nozzle perf tests"
26+
python ci/perf.py --run nozzle --duration ${duration}

‎ci/orca.conf‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[nozzle]
2-
standalone = 1
3-
# search_heads = 1
4-
# indexers = 3
5-
# hec_load_balancers = 1
6-
log_token = 1CB57F19-DC23-419A-8EDA-BA545DD3674D
2+
# standalone = 1
3+
search_heads = 1
4+
indexers = 3
5+
hec_load_balancers = 1
6+
log_token = 00000000-0000-0000-0000-000000000000
77
# perf = true
88
# memory = 8
99
# cpu = 8

‎ci/perf.py‎

Lines changed: 72 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,97 @@
11
import subprocess
22
import argparse
33
import time
4+
import os
45

56

67
nozzle_perf_params = [
7-
{"hec-workers": "1", "hec-batch-size": "100"},
8-
{"hec-workers": "2", "hec-batch-size": "100"},
9-
{"hec-workers": "4", "hec-batch-size": "100"},
10-
{"hec-workers": "8", "hec-batch-size": "100"},
11-
{"hec-workers": "16", "hec-batch-size": "100"},
12-
{"hec-workers": "32", "hec-batch-size": "100"},
13-
{"hec-workers": "1", "hec-batch-size": "1000"},
14-
{"hec-workers": "2", "hec-batch-size": "1000"},
15-
{"hec-workers": "4", "hec-batch-size": "1000"},
16-
{"hec-workers": "8", "hec-batch-size": "1000"},
17-
{"hec-workers": "16", "hec-batch-size": "1000"},
18-
{"hec-workers": "32", "hec-batch-size": "1000"},
8+
{'hec-workers': '1', 'hec-batch-size': '100'},
9+
{'hec-workers': '2', 'hec-batch-size': '100'},
10+
{'hec-workers': '4', 'hec-batch-size': '100'},
11+
{'hec-workers': '8', 'hec-batch-size': '100'},
12+
{'hec-workers': '16', 'hec-batch-size': '100'},
13+
{'hec-workers': '32', 'hec-batch-size': '100'},
14+
{'hec-workers': '1', 'hec-batch-size': '1000'},
15+
{'hec-workers': '2', 'hec-batch-size': '1000'},
16+
{'hec-workers': '4', 'hec-batch-size': '1000'},
17+
{'hec-workers': '8', 'hec-batch-size': '1000'},
18+
{'hec-workers': '16', 'hec-batch-size': '1000'},
19+
{'hec-workers': '32', 'hec-batch-size': '1000'},
1920
]
2021

2122
nozzle_perf_suites = [
2223
{
23-
"message-type": "s256byte",
24-
"extra-fields": "message_type:s256byte",
25-
"cases": nozzle_perf_params,
24+
'message-type': 's256byte',
25+
'extra-fields': 'message_type:s256byte',
26+
'cases': nozzle_perf_params,
2627
},
2728
{
28-
"message-type": "s1kbyte",
29-
"extra-fields": "message_type:s1kbyte",
30-
"cases": nozzle_perf_params,
29+
'message-type': 's1kbyte',
30+
'extra-fields': 'message_type:s1kbyte',
31+
'cases': nozzle_perf_params,
3132
},
3233
{
33-
"message-type": "uns256byte",
34-
"extra-fields": "message_type:uns256byte",
35-
"cases": nozzle_perf_params,
34+
'message-type': 'uns256byte',
35+
'extra-fields': 'message_type:uns256byte',
36+
'cases': nozzle_perf_params,
3637
},
3738
{
38-
"message-type": "uns1kbyte",
39-
"extra-fields": "message_type:uns1kbyte",
40-
"cases": nozzle_perf_params,
39+
'message-type': 'uns1kbyte',
40+
'extra-fields': 'message_type:uns1kbyte',
41+
'cases': nozzle_perf_params,
4142
},
4243
]
4344

4445

4546
def run_nozzle_perf(config):
4647
for suite in nozzle_perf_suites:
47-
for case in suite["cases"]:
48-
kvs = ",".join("{}:{}".format(k, v) for k, v in case.iteritems())
49-
extra_fields = "{},{}".format(suite["extra-fields"], kvs)
48+
for case in suite['cases']:
49+
kvs = ','.join('{}:{}'.format(k, v) for k, v in case.iteritems())
50+
extra_fields = '{},{}'.format(suite['extra-fields'], kvs)
5051

5152
cmd = [
52-
"./splunk-firehose-nozzle",
53-
"--api-endpoint", config["api-endpoint"],
54-
"--user", config["user"],
55-
"--password", config["password"],
56-
"--splunk-host", config["splunk-host"],
57-
"--splunk-token", config["splunk-token"],
58-
"--splunk-index", config["splunk-index"],
59-
"--hec-workers", case["hec-workers"],
60-
"--hec-batch-size", case["hec-batch-size"],
61-
"--events", "ContainerMetric,CounterEvent,Error,HttpStart,HttpStartStop,HttpStop,LogMessage,ValueMetric",
62-
"--extra-fields", extra_fields,
63-
"--add-app-info",
64-
"--enable-event-tracing",
65-
"--skip-ssl-validation-cf",
66-
"--skip-ssl-validation-splunk",
53+
'./splunk-firehose-nozzle',
54+
'--api-endpoint', config['api-endpoint'],
55+
'--user', config['user'],
56+
'--password', config['password'],
57+
'--splunk-host', config['splunk-host'],
58+
'--splunk-token', config['splunk-token'],
59+
'--splunk-index', config['splunk-index'],
60+
'--hec-workers', case['hec-workers'],
61+
'--hec-batch-size', case['hec-batch-size'],
62+
'--events', 'ContainerMetric,CounterEvent,Error,HttpStart,HttpStartStop,HttpStop,LogMessage,ValueMetric',
63+
'--extra-fields', extra_fields,
64+
'--add-app-info',
65+
'--enable-event-tracing',
66+
'--skip-ssl-validation-cf',
67+
'--skip-ssl-validation-splunk',
6768
]
68-
print " ".join(cmd)
69-
execute(cmd, config["duration"])
69+
print ' '.join(cmd)
70+
execute(cmd, config['duration'])
7071

7172

7273
def execute(cmd, duration):
7374
has_error = False
7475
while 1:
7576
try:
7677
start = time.time()
77-
print "start {} {}".format(" ".join(cmd), start)
78+
print 'start {} {}'.format(' '.join(cmd), start)
7879
out, err = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
7980
except Exception as e:
8081
has_error = True
8182
print e
8283
else:
8384
has_error = False
84-
print "out:", out
85-
print "err:", err
85+
print 'out:', out
86+
print 'err:', err
8687
finally:
8788
end = time.time()
8889
last = end - start
89-
print "end {} {} duration={}".format(" ".join(cmd), end, last)
90+
print 'end {} {} duration={}'.format(' '.join(cmd), end, last)
9091

9192
# If we run too short, rerun it
9293
if last < 0.8 * duration:
93-
print "run too short, retry..."
94+
print 'run too short, retry...'
9495
time.sleep(1)
9596
elif has_error:
9697
pass
@@ -100,43 +101,43 @@ def execute(cmd, duration):
100101

101102
def run_trafficcontroller(duration):
102103
for suite in nozzle_perf_suites:
103-
for case in suite["cases"]:
104-
ip = "pcf_{}_{}".format(
105-
suite["message-type"],
106-
"_".join("{}-{}".format(k, v) for k, v in case.iteritems()))
104+
for case in suite['cases']:
105+
ip = 'pcf_{}_{}'.format(
106+
suite['message-type'],
107+
'_'.join('{}-{}'.format(k, v) for k, v in case.iteritems()))
107108
cmd = [
108-
"./trafficcontroller",
109-
"--config", "loggregator_trafficcontroller.json",
110-
"--disableAccessControl",
111-
"--duration", str(duration),
112-
"--message-type", suite["message-type"],
113-
"--ip", ip,
109+
'./trafficcontroller',
110+
'--config', 'loggregator_trafficcontroller.json',
111+
'--disableAccessControl',
112+
'--duration', str(duration),
113+
'--message-type', suite['message-type'],
114+
'--ip', ip,
114115
]
115116

116117
execute(cmd, duration)
117118
time.sleep(10)
118119

119120

120121
def main():
121-
parser = argparse.ArgumentParser(description="Nozzle perf test driver")
122-
parser.add_argument("--run", dest="run", required=True, help="nozzle or trafficcontroller")
123-
parser.add_argument("--duration", dest="duration", type=int, required=True, help="how long to run in seconds")
122+
parser = argparse.ArgumentParser(description='Nozzle perf test driver')
123+
parser.add_argument('--run', dest='run', required=True, help='nozzle or trafficcontroller')
124+
parser.add_argument('--duration', dest='duration', type=int, required=True, help='how long to run in seconds')
124125
args = parser.parse_args()
125126

126-
if args.run == "nozzle":
127+
if args.run == 'nozzle':
127128
config = {
128-
"api-endpoint": "http://ghost:9911",
129-
"user": "admin",
130-
"password": "admin",
131-
"splunk-host": "https://localhost:8088",
132-
"splunk-token": "1CB57F19-DC23-419A-8EDA-BA545DD3674D",
133-
"splunk-index": "main",
134-
"duration": args.duration,
129+
'api-endpoint': os.environ.get('API_ENDPOINT', 'http://trafficcontroller:9911'),
130+
'user': os.environ.get('API_USER', 'admin'),
131+
'password': os.environ.get('API_PASSWORD', 'admin'),
132+
'splunk-host': os.environ.get('SPLUNK_HOST', 'https://heclb1:8088'),
133+
'splunk-token': os.environ.get('SPLUNK_TOKEN', '00000000-0000-0000-0000-000000000000'),
134+
'splunk-index': os.environ.get('SPLUNK_INDEX', 'main'),
135+
'duration': args.duration,
135136
}
136137
run_nozzle_perf(config)
137138
else:
138139
run_trafficcontroller(args.duration)
139140

140141

141-
if __name__ == "__main__":
142+
if __name__ == '__main__':
142143
main()

‎ci/services.yml‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,26 @@
11
trafficcontroller:
2-
image: zlchen/trafficcontroller:1.1
2+
image: zlchen/trafficcontroller:1.3
33
hostname: trafficcontroller
44
ports:
55
- "8081"
66
- "9911"
77
restart: always
88

99
nozzle:
10-
image: zlchen/nozzle:1.1
10+
image: zlchen/nozzle:1.3
1111
hostname: nozzle
1212
ports:
1313
- "9912"
1414
environment:
15+
- API_ENDPOINT=http://trafficcontroller:9911
16+
- API_USER=admin
17+
- API_PASSWORD=admin
1518
- SPLUNK_TOKEN=00000000-0000-0000-0000-000000000000
16-
- SPLUNK_HOST=https://so1:8088
19+
- SPLUNK_HOST=https://heclb1:8088
20+
- SPLUNK_INDEX=main
1721
- SKIP_SSL_VALIDATION_CF=true
1822
- SKIP_SSL_VALIDATION_SPLUNK=true
19-
depends_on:
20-
- trafficcontroller
23+
- NOZZLE_BRANCH=develop
24+
links:
25+
- trafficcontroller
2126
restart: always

‎ci/trafficcontroller.dockerfile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM golang:1.8.3
2+
3+
RUN apt-get update && \
4+
apt-get install -y telnet && \
5+
apt-get install -y python
6+
7+
8+
ENV GOPATH /go
9+
RUN go get -d github.com/chenziliang/loggregator; exit 0
10+
WORKDIR /go/src/github.com/chenziliang/loggregator
11+
RUN git checkout feature/firehose-standalone
12+
ENV GOPATH /go/src/github.com/chenziliang/loggregator
13+
RUN cd /go/src/github.com/chenziliang/loggregator && ./scripts/build
14+
WORKDIR /go/src/github.com/chenziliang/loggregator
15+
RUN mv ./bin/trafficcontroller .
16+
RUN git clone https://github.com/cloudfoundry-community/splunk-firehose-nozzle
17+
18+
EXPOSE 9911
19+
EXPOSE 8081
20+
21+
CMD ["/usr/bin/python", "splunk-firehose-nozzle/ci/perf.py", "--run", "trafficcontroller", "--duration", "1200"]

0 commit comments

Comments
 (0)