Skip to content

Commit 611f3d4

Browse files
anjagruenheidanjagruenheid
authored
Introduce templated benchmarks. (#301)
This PR introduces templated benchmarks for BenchBase. In essence, users can define their own query templates that are parametrized with one or more values at runtime. If multiple parameter combinations are possible, we will continue to cycle through all available combinations. A templated benchmark has the following structure: ```xml <templates> <template name="$QueryTemplateName"> <query>$SQLQuery</query> <types> <type>$ParameterType1</type> <type>$ParameterType2</type> </types> <values> <value>$ParameterValueA1</value> <value>$ParameterValueA2</value> ... </values> <values> <value>$ParameterValueB1</value> <value>$ParameterValueB2</value> ... </values> </template> ... <templates> ``` where `$ParameterType` is the integer `java.sql.Types` value (i.e., `INTEGER`, `VARCHAR`, etc.) and each value tag within `values` contains the values for one instantiation of the parameters set in `$SQLQuery`. The SQL query string is read as a `PreparedStatement`, i.e., parameters are defined in the string via a `?` placeholder. An example for a templated benchmark can be found in `data/templated/example.xml`. The file path for the XML template has to be defined in the workload configuration using the `query_templates_file` tag. An example configuration can be found in `config/sqlserver/sample_template_config.xml`. The example can be executed if a loaded TPC-C instance is used as JDBC endpoint. Templated benchmarks are instantiated using `templated` as benchmark class when running BenchBase via the command line.
1 parent 109064e commit 611f3d4

33 files changed

+1775
-43
lines changed

‎.github/workflows/maven.yml

Lines changed: 54 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ jobs:
8989
fail-fast: false
9090
matrix:
9191
# BROKEN: tpch
92-
benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
92+
benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
9393
steps:
9494
- name: Download artifact
9595
uses: actions/download-artifact@v3
@@ -112,7 +112,16 @@ jobs:
112112

113113
- name: Run benchmark
114114
run: |
115-
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
115+
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
116+
# In this case, we load the tpcc data.
117+
if [[ ${{matrix.benchmark}} == templated ]]; then
118+
# Disable synchronous mode for sqlite tpcc data loading to save some time.
119+
java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
120+
# Run the templated benchmark.
121+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
122+
else
123+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
124+
fi
116125
# FIXME: Reduce the error rate so we don't need these overrides.
117126
if [ ${{matrix.benchmark}} == auctionmark ]; then
118127
ERRORS_THRESHOLD=0.02
@@ -134,7 +143,7 @@ jobs:
134143
strategy:
135144
fail-fast: false
136145
matrix:
137-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
146+
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
138147
services:
139148
mariadb: # https://hub.docker.com/_/mariadb
140149
image: mariadb:latest
@@ -176,7 +185,16 @@ jobs:
176185
MARIADB_PORT: ${{ job.services.mariadb.ports[3306] }}
177186
run: |
178187
mysql -h127.0.0.1 -P$MARIADB_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase"
179-
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
188+
189+
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
190+
# In this case, we load the tpcc data.
191+
if [[ ${{matrix.benchmark}} == templated ]]; then
192+
java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
193+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
194+
else
195+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
196+
fi
197+
180198
# FIXME: Reduce the error rate so we don't need these overrides.
181199
if [ ${{matrix.benchmark}} == auctionmark ]; then
182200
ERRORS_THRESHOLD=0.02
@@ -194,7 +212,7 @@ jobs:
194212
strategy:
195213
fail-fast: false
196214
matrix:
197-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
215+
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
198216
services:
199217
mysql: # https://hub.docker.com/_/mysql
200218
image: mysql:latest
@@ -235,7 +253,16 @@ jobs:
235253
MYSQL_PORT: ${{ job.services.mysql.ports[3306] }}
236254
run: |
237255
mysql -h127.0.0.1 -P$MYSQL_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase"
238-
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
256+
257+
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
258+
# In this case, we load the tpcc data.
259+
if [[ ${{matrix.benchmark}} == templated ]]; then
260+
java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
261+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
262+
else
263+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
264+
fi
265+
239266
# FIXME: Reduce the error rate so we don't need these overrides.
240267
if [ ${{matrix.benchmark}} == auctionmark ]; then
241268
ERRORS_THRESHOLD=0.02
@@ -253,7 +280,7 @@ jobs:
253280
strategy:
254281
fail-fast: false
255282
matrix:
256-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
283+
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
257284
services:
258285
postgres: # https://hub.docker.com/_/postgres
259286
image: postgres:latest
@@ -292,7 +319,16 @@ jobs:
292319
run: |
293320
PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists
294321
PGPASSWORD=password createdb -h localhost -U admin benchbase
295-
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
322+
323+
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
324+
# In this case, we load the tpcc data.
325+
if [[ ${{matrix.benchmark}} == templated ]]; then
326+
java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
327+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
328+
else
329+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
330+
fi
331+
296332
# FIXME: Reduce the error rate so we don't need these overrides.
297333
if [ ${{matrix.benchmark}} == auctionmark ]; then
298334
ERRORS_THRESHOLD=0.02
@@ -362,7 +398,7 @@ jobs:
362398
matrix:
363399
# TODO: add more benchmarks
364400
#benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
365-
benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
401+
benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'tpcc', 'templated', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
366402
services:
367403
sqlserver:
368404
image: mcr.microsoft.com/mssql/server:latest
@@ -423,7 +459,15 @@ jobs:
423459
- name: Run benchmark
424460
# Note: user/pass should match those used in sample configs.
425461
run: |
426-
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
462+
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
463+
# In this case, we load the tpcc data.
464+
if [[ ${{matrix.benchmark}} == templated ]]; then
465+
java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
466+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
467+
else
468+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
469+
fi
470+
427471
# FIXME: Reduce the error rate so we don't need these overrides.
428472
if [ ${{matrix.benchmark}} == tatp ]; then
429473
ERRORS_THRESHOLD=0.05

‎config/mariadb/sample_templated_config.xml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<?xml version="1.0"?>
2+
<parameters>
3+
4+
<!-- Connection details -->
5+
<type>MARIADB</type>
6+
<driver>org.mariadb.jdbc.Driver</driver>
7+
<url>jdbc:mariadb://localhost:3306/benchbase?useServerPrepStmts</url>
8+
<username>admin</username>
9+
<password>password</password>
10+
<isolation>TRANSACTION_SERIALIZABLE</isolation>
11+
<batchsize>128</batchsize>
12+
13+
<!-- Note: these example template reuses TPC-C queries and data as a demonstration, but others are possible.
14+
To test this sample as is, you must first run benchbase with parameters `load=true` and
15+
`config=sample_tpcc_config.xml` to load the TPC-C data to the target database.
16+
Other templated benchmarks need to ensure that the data is loaded manually prior to running the benchmark.
17+
-->
18+
19+
<query_templates_file>data/templated/example.xml</query_templates_file>
20+
21+
<!-- The workload -->
22+
<terminals>1</terminals>
23+
<works>
24+
<work>
25+
<time>10</time>
26+
<rate>100</rate>
27+
<weights>30,20,10,30,10</weights>
28+
</work>
29+
</works>
30+
31+
<!-- Select all templates that are executed. -->
32+
<transactiontypes>
33+
<transactiontype>
34+
<name>GetOrder</name>
35+
</transactiontype>
36+
<transactiontype>
37+
<name>GetCust</name>
38+
</transactiontype>
39+
<transactiontype>
40+
<name>GetCustNull</name>
41+
</transactiontype>
42+
<transactiontype>
43+
<name>GetWarehouse</name>
44+
</transactiontype>
45+
<transactiontype>
46+
<name>GetItemByPrice</name>
47+
</transactiontype>
48+
</transactiontypes>
49+
</parameters>

‎config/mysql/sample_templated_config.xml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<?xml version="1.0"?>
2+
<parameters>
3+
4+
<!-- Connection details -->
5+
<type>MYSQL</type>
6+
<driver>com.mysql.cj.jdbc.Driver</driver>
7+
<url>jdbc:mysql://localhost:3306/benchbase?rewriteBatchedStatements=true&amp;allowPublicKeyRetrieval=True&amp;sslMode=DISABLED</url>
8+
<username>admin</username>
9+
<password>password</password>
10+
<isolation>TRANSACTION_SERIALIZABLE</isolation>
11+
<batchsize>128</batchsize>
12+
13+
<!-- Note: these example template reuses TPC-C queries and data as a demonstration, but others are possible.
14+
To test this sample as is, you must first run benchbase with parameters `load=true` and
15+
`config=sample_tpcc_config.xml` to load the TPC-C data to the target database.
16+
Other templated benchmarks need to ensure that the data is loaded manually prior to running the benchmark.
17+
-->
18+
19+
<query_templates_file>data/templated/example.xml</query_templates_file>
20+
21+
<!-- The workload -->
22+
<terminals>1</terminals>
23+
<works>
24+
<work>
25+
<time>10</time>
26+
<rate>100</rate>
27+
<weights>30,20,10,30,10</weights>
28+
</work>
29+
</works>
30+
31+
<!-- Select all templates that are executed. -->
32+
<transactiontypes>
33+
<transactiontype>
34+
<name>GetOrder</name>
35+
</transactiontype>
36+
<transactiontype>
37+
<name>GetCust</name>
38+
</transactiontype>
39+
<transactiontype>
40+
<name>GetCustNull</name>
41+
</transactiontype>
42+
<transactiontype>
43+
<name>GetWarehouse</name>
44+
</transactiontype>
45+
<transactiontype>
46+
<name>GetItemByPrice</name>
47+
</transactiontype>
48+
</transactiontypes>
49+
</parameters>

‎config/plugin.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,5 @@
1717
<plugin name="smallbank">com.oltpbenchmark.benchmarks.smallbank.SmallBankBenchmark</plugin>
1818
<plugin name="hyadapt">com.oltpbenchmark.benchmarks.hyadapt.HYADAPTBenchmark</plugin>
1919
<plugin name="otmetrics">com.oltpbenchmark.benchmarks.otmetrics.OTMetricsBenchmark</plugin>
20+
<plugin name="templated">com.oltpbenchmark.benchmarks.templated.TemplatedBenchmark</plugin>
2021
</plugins>

‎config/postgres/sample_templated_config.xml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
<?xml version="1.0"?>
2+
<parameters>
3+
4+
<!-- Connection details -->
5+
<type>POSTGRES</type>
6+
<driver>org.postgresql.Driver</driver>
7+
<url>jdbc:postgresql://localhost:5432/benchbase?sslmode=disable&amp;ApplicationName=templated&amp;reWriteBatchedInserts=true</url>
8+
<username>admin</username>
9+
<password>password</password>
10+
<isolation>TRANSACTION_SERIALIZABLE</isolation>
11+
<batchsize>128</batchsize>
12+
13+
<!-- Note: these example template reuses TPC-C queries and data as a demonstration, but others are possible.
14+
To test this sample as is, you must first run benchbase with parameters `load=true` and
15+
`config=sample_tpcc_config.xml` to load the TPC-C data to the target database.
16+
Other templated benchmarks need to ensure that the data is loaded manually prior to running the benchmark.
17+
-->
18+
19+
<query_templates_file>data/templated/example.xml</query_templates_file>
20+
21+
<!-- The workload -->
22+
<terminals>1</terminals>
23+
<works>
24+
<work>
25+
<time>10</time>
26+
<rate>100</rate>
27+
<weights>30,20,10,30,10</weights>
28+
</work>
29+
</works>
30+
31+
<!-- Select all templates that are executed. -->
32+
<transactiontypes>
33+
<transactiontype>
34+
<name>GetOrder</name>
35+
</transactiontype>
36+
<transactiontype>
37+
<name>GetCust</name>
38+
</transactiontype>
39+
<transactiontype>
40+
<name>GetCustNull</name>
41+
</transactiontype>
42+
<transactiontype>
43+
<name>GetWarehouse</name>
44+
</transactiontype>
45+
<transactiontype>
46+
<name>GetItemByPrice</name>
47+
</transactiontype>
48+
</transactiontypes>
49+
</parameters>

‎config/sqlite/sample_templated_config.xml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<?xml version="1.0"?>
2+
<parameters>
3+
4+
<!-- Connection details -->
5+
<type>SQLITE</type>
6+
<driver>org.sqlite.JDBC</driver>
7+
<url>jdbc:sqlite:tpcc.db</url>
8+
<isolation>TRANSACTION_SERIALIZABLE</isolation>
9+
<batchsize>128</batchsize>
10+
11+
<!-- Note: these example template reuses TPC-C queries and data as a demonstration, but others are possible.
12+
To test this sample as is, you must first run benchbase with parameters `load=true` and
13+
`config=sample_tpcc_config.xml` to load the TPC-C data to the target database.
14+
Other templated benchmarks need to ensure that the data is loaded manually prior to running the benchmark.
15+
-->
16+
17+
<query_templates_file>data/templated/example.xml</query_templates_file>
18+
19+
<!-- The workload -->
20+
<terminals>1</terminals>
21+
<works>
22+
<work>
23+
<time>10</time>
24+
<rate>100</rate>
25+
<weights>30,20,10,30,10</weights>
26+
</work>
27+
</works>
28+
29+
<!-- Select all templates that are executed. -->
30+
<transactiontypes>
31+
<transactiontype>
32+
<name>GetOrder</name>
33+
</transactiontype>
34+
<transactiontype>
35+
<name>GetCust</name>
36+
</transactiontype>
37+
<transactiontype>
38+
<name>GetCustNull</name>
39+
</transactiontype>
40+
<transactiontype>
41+
<name>GetWarehouse</name>
42+
</transactiontype>
43+
<transactiontype>
44+
<name>GetItemByPrice</name>
45+
</transactiontype>
46+
</transactiontypes>
47+
</parameters>

‎config/sqlite/sample_tpcc_config.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
<!-- Scale factor is the number of warehouses in TPCC -->
1212
<scalefactor>1</scalefactor>
13-
13+
1414
<!-- SQLITE only supports one writer thread -->
1515
<loaderThreads>1</loaderThreads>
1616

0 commit comments

Comments
 (0)