Skip to content

Commit cf0a44f

Browse files
hoffmaddhoffmadd
authored
Merge pull request #54 from levigo/feat/JF-756-AddJS5TimedLoadTest
feat(JF-756): Add JS5 Timed Load Test
2 parents 686eebe + f5cc87a commit cf0a44f

File tree

8 files changed

+300
-47
lines changed

8 files changed

+300
-47
lines changed

‎src/main/java/org/levigo/jadice/server/converterclient/JobCardScheduler.java‎

Lines changed: 42 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -5,43 +5,46 @@
55
import java.util.concurrent.TimeUnit;
66

77
public class JobCardScheduler {
8-
9-
private final static JobCardScheduler INSTANCE = new JobCardScheduler();
10-
11-
private static final class MyShutdownHook extends Thread {
12-
13-
public MyShutdownHook() {
14-
super("ShutdownHook for JobCardScheduler");
15-
}
16-
17-
@Override
18-
public void run() {
19-
getInstance().executor.shutdown();
20-
getInstance().executor.shutdownNow();
21-
}
22-
}
23-
24-
public static JobCardScheduler getInstance() {
25-
return INSTANCE;
26-
}
27-
28-
private JobCardScheduler() {
29-
Runtime.getRuntime().addShutdownHook(new MyShutdownHook());
30-
Preferences.concurrentJobsProperty().addListener((obj, oldValue, newValue) -> {
31-
executor.setCorePoolSize(newValue.intValue());
32-
});
33-
executor.setCorePoolSize(Preferences.concurrentJobsProperty().getValue());
34-
}
35-
36-
private ThreadPoolExecutor executor = new ThreadPoolExecutor(4, 4,
37-
0L, TimeUnit.MILLISECONDS,
38-
new LinkedBlockingQueue<Runnable>());
39-
40-
public void submit(JobCard card) {
41-
executor.submit(card);
42-
}
43-
44-
public void shutdown() {
45-
this.executor.shutdown();
46-
}
8+
9+
private final static JobCardScheduler INSTANCE = new JobCardScheduler();
10+
11+
private static final class MyShutdownHook extends Thread {
12+
13+
public MyShutdownHook() {
14+
super("ShutdownHook for JobCardScheduler");
15+
}
16+
17+
@Override
18+
public void run() {
19+
getInstance().executor.shutdown();
20+
getInstance().executor.shutdownNow();
21+
}
22+
}
23+
24+
public static JobCardScheduler getInstance() {
25+
return INSTANCE;
26+
}
27+
28+
private JobCardScheduler() {
29+
Runtime.getRuntime().addShutdownHook(new MyShutdownHook());
30+
Preferences.concurrentJobsProperty().addListener((obj, oldValue, newValue) -> {
31+
executor.setCorePoolSize(newValue.intValue());
32+
});
33+
executor.setCorePoolSize(Preferences.concurrentJobsProperty().getValue());
34+
}
35+
36+
private ThreadPoolExecutor executor = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
37+
new LinkedBlockingQueue<Runnable>());
38+
39+
public void submit(JobCard card) {
40+
executor.submit(card);
41+
}
42+
43+
public void shutdown() {
44+
this.executor.shutdown();
45+
}
46+
47+
public int getCurrentQueueSize() {
48+
return executor.getQueue().size();
49+
}
4750
}

‎src/main/java/org/levigo/jadice/server/converterclient/util/FilenameGenerator.java‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import java.io.File;
44
import java.io.IOException;
55
import java.util.Map;
6+
import java.util.UUID;
67
import java.util.regex.Matcher;
78

89
import org.apache.log4j.Logger;
@@ -15,48 +16,51 @@
1516
import com.levigo.jadice.server.shared.types.Stream;
1617

1718
public class FilenameGenerator {
18-
19+
1920
private static final Logger LOGGER = Logger.getLogger(FilenameGenerator.class);
20-
21-
private static Analyzer analyzer;
2221

23-
public final static String DEFAULT_PATTERN = PatternKeys.ORIGINAL_FILENAME + "-" + PatternKeys.NUMMER + "."
24-
+ PatternKeys.EXTENSION;
22+
private static Analyzer analyzer;
23+
24+
public final static String DEFAULT_PATTERN = PatternKeys.ORIGINAL_FILENAME + "-" + PatternKeys.NUMMER + "-"
25+
+ PatternKeys.UUID + "." + PatternKeys.EXTENSION;
2526

2627
public static interface PatternKeys {
2728
final String JOB_ID = "%j";
2829
final String ORIGINAL_FILENAME = "%f";
2930
final String EXTENSION = "%e";
3031
final String NUMMER = "%n";
3132
final String TIMESTAMP = "%t";
33+
final String UUID = "%u";
3234
final String PERCENT_SIGN = "%%";
3335
}
3436

3537
public static String generateFilename(Job job, Stream stream, File originalFile, int nmbr) {
3638
String pttrn = Preferences.resultFilenamePatternProperty().getValue();
3739
// Caveat! Matcher.quoteReplacement(...) in order to escape $ and \ signs
38-
40+
3941
if (mustDetermineExtension()) {
4042
final String extension = determineExtension(stream);
4143
pttrn = pttrn.replaceAll(PatternKeys.EXTENSION, Matcher.quoteReplacement(extension));
4244
}
4345
return pttrn.replaceAll(PatternKeys.JOB_ID, Matcher.quoteReplacement(job.getUUID()))//
4446
.replaceAll(PatternKeys.ORIGINAL_FILENAME, Matcher.quoteReplacement(originalFile.getName()))//
4547
.replaceAll(PatternKeys.NUMMER, Integer.toString(nmbr))//
48+
.replaceAll(PatternKeys.UUID, UUID.randomUUID().toString().replace("-", ""))//
4649
.replaceAll(PatternKeys.TIMESTAMP, Long.toString(System.currentTimeMillis()))//
4750
.replaceAll(PatternKeys.PERCENT_SIGN, "%")
4851
// Paranoia checks to prevent from storing in other folders:
4952
.replaceAll("/", "")//
5053
.replace("\\", "");
5154
}
52-
55+
5356
protected static boolean mustDetermineExtension() {
5457
return Preferences.resultFilenamePatternProperty().getValue().contains(PatternKeys.EXTENSION);
5558
}
5659

5760
private static String determineExtension(Stream stream) {
5861
try {
59-
final UncloseableSeekableInputStreamWrapper usis = new UncloseableSeekableInputStreamWrapper(stream.getInputStream());
62+
final UncloseableSeekableInputStreamWrapper usis = new UncloseableSeekableInputStreamWrapper(
63+
stream.getInputStream());
6064
final Map<String, Object> alResults;
6165
try {
6266
usis.seek(0);
@@ -103,6 +107,8 @@ public static String buildExplanationText(String sep) {
103107
s += FilenameGenerator.PatternKeys.TIMESTAMP + ": timestamp";
104108
s += sep;
105109
s += FilenameGenerator.PatternKeys.PERCENT_SIGN + ": %";
110+
s += sep;
111+
s += FilenameGenerator.PatternKeys.UUID + ": generated UUID";
106112
return s;
107113
}
108114
}

‎src/test/java/org/levigo/jadice/server/converterclient/JS5TimedLoadTest.java‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
package org.levigo.jadice.server.converterclient;
2+
3+
import java.io.File;
4+
import java.io.FileInputStream;
5+
import java.io.FileOutputStream;
6+
import java.io.IOException;
7+
import java.nio.channels.FileChannel;
8+
import java.time.Duration;
9+
import java.util.ArrayList;
10+
import java.util.Collections;
11+
import java.util.Date;
12+
import java.util.HashMap;
13+
import java.util.List;
14+
import java.util.Map;
15+
import java.util.UUID;
16+
import java.util.concurrent.TimeUnit;
17+
18+
import org.junit.Before;
19+
import org.junit.jupiter.api.Disabled;
20+
import org.junit.jupiter.api.Test;
21+
import org.levigo.jadice.server.converterclient.configurations.WorkflowConfiguration;
22+
23+
public class JS5TimedLoadTest {
24+
25+
private long testDurationMS = TimeUnit.MINUTES.toMillis(15);
26+
27+
private String templateID = "x2tiff (DOCP)";
28+
private File testdataFolder = new File(System.getProperty("user.dir"), "/testdata/perf-test/image");
29+
30+
// private String templateID = "mail2pdf (lo)";
31+
// private File testdataFolder = new File(System.getProperty("user.dir"),
32+
// "/testdata/perf-test/email");
33+
34+
private int targetQueueSize = 40;
35+
36+
private JobCardScheduler scheduler = JobCardScheduler.getInstance();
37+
38+
private long sleepTimeBetweenCheckMS = 1000;
39+
40+
private List<File> testdataImageFiles = new ArrayList<>();
41+
private int currentTestdataIndex = 0;
42+
43+
private List<JobCard> createdJobCards = new ArrayList<>();
44+
private int currentItemCounter = 0;
45+
46+
private String serverLocation = "tcp://localhost:61616";
47+
private WorkflowConfiguration config;
48+
49+
@Disabled // TODO: Enable on your local machine
50+
@Test
51+
public void performLoadTest() throws Exception {
52+
init();
53+
54+
long totalDurationMS = System.currentTimeMillis();
55+
config = JobCardFactory.getInstance().getConfiguration(templateID);
56+
if (config == null) {
57+
throw new IllegalArgumentException("Configuration \"" + templateID + "\" unknown");
58+
}
59+
60+
Map<String, Object> info = new HashMap<>();
61+
info.put("desiredRuntime [sec]", (testDurationMS / 1000));
62+
info.put("templateID", templateID);
63+
info.put("targetQueueSize", targetQueueSize);
64+
info.put("startTime", new Date(System.currentTimeMillis()));
65+
66+
System.out.println("Performing load test...\n" + info);
67+
68+
doPerformTest();
69+
70+
totalDurationMS = System.currentTimeMillis() - totalDurationMS;
71+
72+
// Add some statistics
73+
info.put("createdItemCount", currentItemCounter);
74+
info.put("createdJobCount", createdJobCards.size());
75+
76+
double timePerJob = Double.valueOf(totalDurationMS).doubleValue()
77+
/ Double.valueOf(createdJobCards.size()).doubleValue();
78+
double timePerItem = Double.valueOf(totalDurationMS).doubleValue()
79+
/ Double.valueOf(currentItemCounter).doubleValue();
80+
double jobsPerMinute = Double.valueOf(60000) / timePerJob;
81+
double itemsPerMinute = Double.valueOf(60000) / timePerItem;
82+
double itemsPerHour = itemsPerMinute * 60;
83+
84+
info.put("timePerItem", timePerItem);
85+
info.put("itemsPerMinute", itemsPerMinute);
86+
info.put("itemsPerHour", itemsPerHour);
87+
info.put("timePerJob", timePerJob);
88+
info.put("jobsPerMinute", jobsPerMinute);
89+
90+
// Log finished message
91+
StringBuilder sb = new StringBuilder();
92+
for (String key : info.keySet()) {
93+
if (sb.length() > 0) {
94+
sb.append(System.lineSeparator());
95+
}
96+
sb.append(key);
97+
sb.append("=");
98+
sb.append(info.get(key));
99+
}
100+
101+
String msg = String.format("Finished load test:\n===========================\n%s\n===========================",
102+
sb.toString());
103+
104+
System.out.println(msg);
105+
106+
System.out.println("Test ended");
107+
}
108+
109+
private void init() {
110+
System.out.println("Reading testdata folder...");
111+
// Read all file objects from folder so we can easily choose later on via rotating index
112+
testdataImageFiles.clear();
113+
if (testdataFolder.exists()) {
114+
File[] sub = testdataFolder.listFiles();
115+
if (null != sub) {
116+
for (File s : sub) {
117+
testdataImageFiles.add(s);
118+
}
119+
}
120+
}
121+
}
122+
123+
@Before
124+
private void beforeEachTest() {
125+
currentItemCounter = 0;
126+
createdJobCards.clear();
127+
currentTestdataIndex = 0;
128+
}
129+
130+
private void doPerformTest() throws Exception {
131+
boolean shallRun = true;
132+
long start = System.currentTimeMillis();
133+
long end = start + testDurationMS;
134+
135+
System.out.println("Running performance test. Will run until: " + new Date(end));
136+
137+
while (shallRun) {
138+
int currentQueueSize = scheduler.getCurrentQueueSize();
139+
140+
if (currentQueueSize < targetQueueSize) {
141+
System.out.println("Creating new jobs to fill up queue...");
142+
143+
while (currentQueueSize < targetQueueSize) {
144+
145+
System.out.println("Creating next job");
146+
JobCard jobCard = createNextJob();
147+
createdJobCards.add(jobCard);
148+
149+
currentItemCounter++;
150+
151+
currentQueueSize = scheduler.getCurrentQueueSize();
152+
}
153+
}
154+
155+
if (System.currentTimeMillis() > end) {
156+
shallRun = false;
157+
} else {
158+
Thread.sleep(sleepTimeBetweenCheckMS);
159+
System.out.println(String.format(
160+
"Running... Created job count: %s / created item count %s / runtime left %s min", createdJobCards.size(),
161+
currentItemCounter, Duration.ofMillis(end - System.currentTimeMillis()).toMinutes()));
162+
}
163+
}
164+
165+
System.out.println("Loop finished");
166+
}
167+
168+
private JobCard createNextJob() throws Exception {
169+
170+
if (currentTestdataIndex > testdataImageFiles.size() - 2) {
171+
currentTestdataIndex = 0;
172+
} else {
173+
currentTestdataIndex++;
174+
}
175+
176+
File chosenFile = testdataImageFiles.get(currentTestdataIndex);
177+
178+
JobCard jobCard = JobCardFactory.getInstance().createAndSubmitJobCard(chosenFile, serverLocation, config,
179+
Collections.emptySet());
180+
181+
return jobCard;
182+
}
183+
184+
private void processResult(JobCard jobCard) throws IOException, InterruptedException {
185+
List<File> result = new ArrayList<File>();
186+
187+
jobCard.job.waitForTermination(-1, TimeUnit.SECONDS);
188+
189+
while (!jobCard.isResultsCompleted()) {
190+
Thread.sleep(100);
191+
}
192+
193+
for (File from : jobCard.getResults()) {
194+
String outFilename = "./target/out/" + /* inFile.getName() + */ "-out-" + UUID.randomUUID().toString() + "-"
195+
+ from.getName();
196+
File outFile = new File(System.getProperty("user.dir"), outFilename);
197+
198+
copyFile(from, outFile);
199+
result.add(outFile);
200+
}
201+
}
202+
203+
public static void copyFile(File from, File to) throws IOException {
204+
final FileInputStream fis = new FileInputStream(from);
205+
try {
206+
final FileOutputStream fos = new FileOutputStream(to);
207+
try {
208+
final FileChannel inChannel = fis.getChannel();
209+
final FileChannel outChannel = fos.getChannel();
210+
inChannel.transferTo(0, inChannel.size(), outChannel);
211+
} finally {
212+
fos.close();
213+
}
214+
} finally {
215+
fis.close();
216+
}
217+
}
218+
}

0 commit comments

Comments
 (0)