Skip to content

Commit c28d174

Browse files
rmuirrmuir
authored
script to run microbenchmarks across different ec2 instance types (#12787)
1 parent d458356 commit c28d174

File tree

9 files changed

+351
-0
lines changed

9 files changed

+351
-0
lines changed

‎dev-tools/aws-jmh/Makefile‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
.PHONY: test clean env
2+
.DELETE_ON_ERROR:
3+
4+
# python interpreter to use
5+
export PYTHON=python3
6+
# improve ansible output
7+
export ANSIBLE_CALLBACK_RESULT_FORMAT=yaml
8+
# reasonable default verbosity, crank to debug issues
9+
export ANSIBLE_VERBOSITY=0
10+
# pipeline operations over ssh
11+
export ANSIBLE_PIPELINING=true
12+
# show breakdown of task times
13+
export ANSIBLE_CALLBACKS_ENABLED=profile_tasks
14+
# not useful since they are throwaways
15+
export ANSIBLE_HOST_KEY_CHECKING=false
16+
# run all machines in parallel
17+
export ANSIBLE_FORKS=10
18+
19+
ifeq ($(AWS_ACCESS_KEY_ID),)
20+
$(error AWS_ACCESS_KEY_ID is not set)
21+
endif
22+
23+
ifeq ($(AWS_SECRET_ACCESS_KEY),)
24+
$(error AWS_SECRET_ACCESS_KEY is not set)
25+
endif
26+
27+
ifeq ($(PATCH_BRANCH),)
28+
$(error PATCH_BRANCH is not set)
29+
endif
30+
31+
# runs benchmark across AWS instances
32+
# always destroys anything created out there
33+
test: env
34+
bash -c "set -ex; trap 'build/.env/bin/ansible-playbook destroy.yml' EXIT; build/.env/bin/ansible-playbook playbook.yml"
35+
36+
clean:
37+
rm -rf build
38+
39+
# keep venv in sync with requirements.txt
40+
env: build/.env/bin/activate
41+
build/.env/bin/activate: requirements.txt
42+
rm -rf build/.env
43+
$(PYTHON) -m venv build/.env
44+
build/.env/bin/pip install --no-input -U -r requirements.txt
45+
touch build/.env/bin/activate

‎dev-tools/aws-jmh/README.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Runs lucene microbenchmarks across a variety of CPUs in EC2.
2+
3+
Example:
4+
5+
export AWS_ACCESS_KEY_ID=xxxxx
6+
export AWS_SECRET_ACCESS_KEY=yyyy
7+
make PATCH_BRANCH=rmuir:some-speedup
8+
9+
Results file will be in build/report.txt
10+
11+
You can also pass additional JMH args if you want:
12+
13+
make PATCH_BRANCH=rmuir:some-speedup JMH_ARGS='float -p size=756'

‎dev-tools/aws-jmh/cloud-init.yml.j2‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#cloud-config
2+
package_upgrade: false
3+
package_update: false
4+
system_info:
5+
default_user:
6+
name: {{ user }}
7+
ssh_authorized_keys:
8+
- {{ lookup('file', ssh_key + '.pub') }}

‎dev-tools/aws-jmh/cloudformation.yml.j2‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
AWSTemplateFormatVersion: 2010-09-09
2+
Resources:
3+
4+
SecurityGroupIngress:
5+
Type: AWS::EC2::SecurityGroup
6+
Properties:
7+
VpcId: {{ vpc_info.vpcs[0].id }}
8+
GroupDescription: Allow inbound SSH traffic
9+
SecurityGroupIngress:
10+
- IpProtocol: "tcp"
11+
FromPort: 22
12+
ToPort: 22
13+
CidrIp: "0.0.0.0/0"
14+
Tags:
15+
- Key: Name
16+
Value: {{ stack_name }}-inbound-ssh
17+
18+
SecurityGroupEgress:
19+
Type: AWS::EC2::SecurityGroup
20+
Properties:
21+
VpcId: {{ vpc_info.vpcs[0].id }}
22+
GroupDescription: Allow all outbound traffic
23+
SecurityGroupEgress:
24+
- IpProtocol: "-1"
25+
CidrIp: "0.0.0.0/0"
26+
Tags:
27+
- Key: Name
28+
Value: {{ stack_name }}-outbound-all
29+
30+
{% for instance in instances %}
31+
{{ instance.name }}Instance:
32+
Type: AWS::EC2::Instance
33+
DependsOn:
34+
- SecurityGroupIngress
35+
- SecurityGroupEgress
36+
Properties:
37+
InstanceType: {{ instance.type }}
38+
InstanceInitiatedShutdownBehavior: terminate
39+
ImageId: {{ instance.ami }}
40+
SecurityGroupIds:
41+
- !Ref SecurityGroupIngress
42+
- !Ref SecurityGroupEgress
43+
UserData: {{ lookup('template', 'cloud-init.yml.j2') | b64encode }}
44+
Tags:
45+
- Key: Name
46+
Value: {{ stack_name }}-{{ instance.name }}
47+
- Key: ShortName
48+
Value: {{ instance.name }}
49+
{% endfor %}

‎dev-tools/aws-jmh/destroy.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
- hosts: localhost
2+
gather_facts: false
3+
tasks:
4+
- name: Destroy cloudformation stack
5+
cloudformation:
6+
stack_name: "{{ stack_name }}"
7+
region: "{{ region }}"
8+
state: absent
9+
delegate_to: localhost

‎dev-tools/aws-jmh/group_vars/all.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# cloudformation stack name
2+
stack_name: lucene-jmh
3+
4+
# region to launch instances, make sure they are supported!
5+
region: us-east-1
6+
7+
# ssh keypair to use for accessing instances
8+
ssh_key: ~/.ssh/id_ed25519
9+
10+
# user to create remotely
11+
user: lucene
12+
13+
# ami to use
14+
x86_ami: ami-04cb4ca688797756f
15+
arm_ami: ami-06f9c0b2ce386dda7
16+
17+
# instance types to spin up
18+
instances:
19+
- name: haswell
20+
type: c4.large
21+
ami: "{{ x86_ami }}"
22+
- name: cascadelake
23+
type: c5.large
24+
ami: "{{ x86_ami }}"
25+
- name: icelake
26+
type: c6i.large
27+
ami: "{{ x86_ami }}"
28+
- name: sapphirerapids
29+
type: c7i.large
30+
ami: "{{ x86_ami }}"
31+
- name: zen2
32+
type: c5a.large
33+
ami: "{{ x86_ami }}"
34+
- name: zen3
35+
type: c6a.large
36+
ami: "{{ x86_ami }}"
37+
- name: zen4
38+
type: c7a.large
39+
ami: "{{ x86_ami }}"
40+
- name: graviton2
41+
type: c6g.large
42+
ami: "{{ arm_ami }}"
43+
- name: graviton3
44+
type: c7g.large
45+
ami: "{{ arm_ami }}"
46+
47+
jdk_version: 21.0.1
48+
x86_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-x64_bin.tar.gz"
49+
arm_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-aarch64_bin.tar.gz"
50+
jdk_url: "{{ arm_jdk_url if ansible_architecture == 'aarch64' else x86_jdk_url }}"
51+
52+
main_branch: "apache:main"
53+
patch_branch: "{{ lookup('env', 'PATCH_BRANCH') }}"
54+
55+
assemble_command: ./gradlew -p lucene/benchmark-jmh assemble
56+
jmh_command: java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-10.0.0-SNAPSHOT.jar -rf text
57+
jmh_args: "{{ lookup('env', 'JMH_ARGS', default='-p size=1024') }}"
58+
59+
# services that cause noise and don't help us
60+
disable_services:
61+
- atd.service
62+
- logrotate.timer
63+
- sysstat-collect.timer
64+
- sysstat-summary.timer
65+
- systemd-tmpfiles-clean.timer

‎dev-tools/aws-jmh/playbook.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
2+
# spin up aws hardware
3+
- hosts: localhost
4+
gather_facts: false
5+
tasks:
6+
- name: Lookup default VPC
7+
ec2_vpc_net_info:
8+
region: "{{ region }}"
9+
filters:
10+
isDefault: true
11+
register: vpc_info
12+
13+
- name: Create cloudformation stack
14+
cloudformation:
15+
stack_name: "{{ stack_name }}"
16+
region: "{{ region }}"
17+
template_body: "{{ lookup('template', 'cloudformation.yml.j2') }}"
18+
19+
- name: Gather instance details
20+
ec2_instance_info:
21+
region: "{{ region }}"
22+
filters:
23+
instance-state-name: ["pending", "running"]
24+
"tag:aws:cloudformation:stack-name": "{{ stack_name }}"
25+
register: ec2_instances
26+
27+
- name: Add instances to inventory
28+
add_host:
29+
name: "{{ item['tags']['ShortName'] }}"
30+
ansible_host: "{{ item['public_ip_address'] }}"
31+
ansible_user: "{{ user }}"
32+
ansible_ssh_private_key_file: "{{ ssh_key }}"
33+
args: "{{ item }}"
34+
with_items: "{{ ec2_instances.instances }}"
35+
36+
- hosts: all
37+
gather_facts: false
38+
tasks:
39+
- name: Wait for connection
40+
wait_for_connection:
41+
42+
- name: Gather facts
43+
setup:
44+
45+
- name: Configure kernel
46+
command:
47+
argv:
48+
- grubby
49+
- --remove-args=selinux_1 security=selinux quiet
50+
- --args=mitigations=0 random.trust_cpu=1 loglevel=7 selinux=0
51+
- --update-kernel=ALL
52+
become: true
53+
54+
- name: Install packages
55+
package:
56+
name:
57+
- git
58+
become: true
59+
60+
- name: Disable noisy services
61+
command:
62+
cmd: "systemctl disable --now {{ item }}"
63+
loop: "{{ disable_services }}"
64+
failed_when: false
65+
become: true
66+
67+
- name: Reboot machine
68+
reboot:
69+
become: true
70+
71+
- name: Download JDK
72+
unarchive:
73+
src: "{{ jdk_url }}"
74+
dest: .
75+
remote_src: true
76+
77+
- name: Configure JDK
78+
blockinfile:
79+
path: .bashrc
80+
block: |
81+
export JAVA_HOME=/home/{{ user }}/jdk-{{ jdk_version }}
82+
export PATH=$JAVA_HOME/bin:$PATH
83+
84+
- name: Configure Gradle
85+
blockinfile:
86+
path: .gradle/gradle.properties
87+
create: true
88+
block: |
89+
org.gradle.daemon=false
90+
91+
- name: Checkout main
92+
git:
93+
repo: "[email protected]:{{ (main_branch | split(':'))[0] }}/lucene.git"
94+
version: "{{ (main_branch | split(':'))[1] }}"
95+
dest: main
96+
accept_newhostkey: true
97+
depth: 1
98+
99+
- name: Checkout patch
100+
git:
101+
repo: "[email protected]:{{ (patch_branch | split(':'))[0] }}/lucene.git"
102+
version: "{{ (patch_branch | split(':'))[1] }}"
103+
dest: patch
104+
accept_newhostkey: true
105+
depth: 1
106+
107+
- name: Assemble Sources
108+
command:
109+
cmd: "{{ assemble_command }}"
110+
chdir: "{{ item }}"
111+
loop:
112+
- main
113+
- patch
114+
115+
- name: Run benchmark
116+
command:
117+
cmd: "{{ jmh_command }} -rff ~/{{ item }}.txt {{ jmh_args }}"
118+
chdir: "{{ item }}"
119+
loop:
120+
- main
121+
- patch
122+
123+
- name: Read main results
124+
command:
125+
cmd: "cat main.txt"
126+
register: main_out
127+
128+
- name: Read patch results
129+
command:
130+
cmd: "cat patch.txt"
131+
register: patch_out
132+
133+
- name: Write Report
134+
copy:
135+
dest: "build/{{ inventory_hostname }}.log"
136+
content: |
137+
138+
{{ inventory_hostname }}: `{{ ansible_processor }}`
139+
140+
main
141+
```
142+
{{ main_out.stdout }}
143+
```
144+
patch
145+
```
146+
{{ patch_out.stdout }}
147+
```
148+
delegate_to: localhost
149+
150+
- hosts: localhost
151+
gather_facts: false
152+
tasks:
153+
- name: Create combined report
154+
shell:
155+
cmd: "cat build/*.log > build/report.txt"

‎dev-tools/aws-jmh/requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# automate botnet with ansible
2+
ansible<9
3+
# aws api
4+
boto3<2

‎gradle/validation/rat-sources.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ allprojects {
8282
// projects. Include their sources in the scan.
8383
include "buildSrc/src/**"
8484
include "dev-tools/missing-doclet/src/**"
85+
86+
// do not let RAT attempt to scan a python venv, it gets lost and confused...
87+
exclude "dev-tools/aws-jmh/build/**"
8588
break
8689

8790
case ":lucene:analysis:morfologik":

0 commit comments

Comments
 (0)