Skip to content

Commit 27aad2d

Browse files
David Grievekcpeppe
David Grieve
and
kcpeppe
authored
GCToolKit#addDataSourceParser (#288)
* refactor: fix up mapping of aggregators to channels * add method GCToolkit#addDataSourceParser * Add GCToolKit#LOG_DEBUG_MESSAGE * improve debug messages * clean up formatting of debug message * s/producesEvents/eventsProduced/ --------- Co-authored-by: Kirk Pepperdine <[email protected]>
1 parent 35cee52 commit 27aad2d

23 files changed

+366
-104
lines changed

‎api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java‎

Lines changed: 126 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
package com.microsoft.gctoolkit;
44

55
import com.microsoft.gctoolkit.aggregator.Aggregation;
6+
import com.microsoft.gctoolkit.aggregator.Aggregator;
7+
import com.microsoft.gctoolkit.aggregator.EventSource;
68
import com.microsoft.gctoolkit.io.DataSource;
79
import com.microsoft.gctoolkit.io.GCLogFile;
810
import com.microsoft.gctoolkit.io.RotatingGCLogFile;
@@ -14,16 +16,21 @@
1416
import com.microsoft.gctoolkit.message.JVMEventChannel;
1517

1618
import java.io.IOException;
19+
import java.lang.reflect.Constructor;
1720
import java.lang.reflect.InvocationTargetException;
21+
import java.lang.reflect.Parameter;
1822
import java.util.ArrayList;
1923
import java.util.Arrays;
2024
import java.util.HashSet;
2125
import java.util.List;
2226
import java.util.Optional;
2327
import java.util.ServiceConfigurationError;
2428
import java.util.ServiceLoader;
29+
import java.util.Set;
30+
import java.util.function.Supplier;
2531
import java.util.logging.Level;
2632
import java.util.logging.Logger;
33+
import java.util.stream.Collector;
2734
import java.util.stream.Collectors;
2835

2936
import static java.lang.Class.forName;
@@ -35,6 +42,38 @@ public class GCToolKit {
3542

3643
private static final Logger LOGGER = Logger.getLogger(GCToolKit.class.getName());
3744

45+
private static final String GCTOOLKIT_DEBUG = System.getProperty("gctoolkit.debug");
46+
private static final boolean DEBUGGING = GCTOOLKIT_DEBUG != null;
47+
48+
// returns true if gctoolkit.debug is set to "all" or contains "className", but does not contain "-className"
49+
private static boolean isDebugging(String className) {
50+
return DEBUGGING
51+
&& (GCTOOLKIT_DEBUG.isEmpty()
52+
|| ((GCTOOLKIT_DEBUG.contains("all") || GCTOOLKIT_DEBUG.contains(className))
53+
&& !GCTOOLKIT_DEBUG.contains("-" + className)));
54+
}
55+
56+
/**
57+
* Print a debug message to System.out if gctoolkit.debug is empty, is set to "all",
58+
* or contains "className" but does not contain "-className".
59+
* For example, to enable debug logging for all classes except UnifiedG1GCParser:
60+
* <code>-Dgctoolkit.debug=all,-com.microsoft.gctoolkit.parser.UnifiedG1GCParser</code>
61+
*
62+
* @param message Supplies the message to log. If null, nothing will be logged.
63+
*/
64+
public static void LOG_DEBUG_MESSAGE(Supplier<String> message) {
65+
if (DEBUGGING && message != null) {
66+
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
67+
String methodName = stackTrace[2].getMethodName();
68+
String className = stackTrace[2].getClassName();
69+
String fileName = stackTrace[2].getFileName();
70+
int lineNumber = stackTrace[2].getLineNumber();
71+
if (isDebugging(className)) {
72+
System.out.println(String.format("DEBUG: %s.%s(%s:%d): %s", className, methodName, fileName, lineNumber, message.get()));
73+
}
74+
}
75+
}
76+
3877
private final HashSet<DataSourceParser> registeredDataSourceParsers = new HashSet<>();
3978
private List<Aggregation> registeredAggregations;
4079
private JVMEventChannel jvmEventChannel = null;
@@ -71,10 +110,10 @@ public void loadAggregationsFromServiceLoader() {
71110
ServiceLoader.load(Aggregation.class)
72111
.stream()
73112
.map(ServiceLoader.Provider::get)
74-
.forEach(registeredAggregations::add);
75-
//Useful for debugging
76-
if ( Level.FINER.equals(LOGGER.getLevel()))
77-
registeredAggregations.forEach(a -> LOGGER.log(Level.FINER, "Registered " + a.toString()));
113+
.forEach(aggregation -> {
114+
registeredAggregations.add(aggregation);
115+
LOG_DEBUG_MESSAGE(() -> "ServiceLoader provided: " + aggregation.getClass().getName());
116+
});
78117
}
79118

80119
/**
@@ -154,19 +193,41 @@ private void loadJVMEventChannel() {
154193
}
155194
}
156195

196+
/**
197+
* This method allows full control over which DataSourceParsers are used to parse the DataSource.
198+
* This method should be called before the {@link #analyze(DataSource)} method.
199+
* DataSourceParsers loaded by this method are used in place of those that are ordinarily loaded via
200+
* the service provider interface.
201+
* Use the {@link #addDataSourceParser(DataSourceParser)} method to load a DataSourceParser in addition
202+
* to those loaded by the service provider interface.
203+
* @param dataSourceParser An implementation of DataSourceParser that will be used to parse the DataSource.
204+
*/
157205
public void loadDataSourceParser(DataSourceParser dataSourceParser) {
158206
registeredDataSourceParsers.add(dataSourceParser);
159207
}
160208

161-
private void loadDataSourceParsers(Diary diary) {
209+
private List<DataSourceParser> additiveParsers = new ArrayList<>();
210+
211+
/**
212+
* Add a DataSourceParser to be used to parse a DataSource. The DataSourceParser will be used in addition
213+
* to those loaded by the service provider interface. This method should be called before the
214+
* {@link #analyze(DataSource)} method.
215+
* @param dataSourceParser An implementation of DataSourceParser that will be used to parse the DataSource.
216+
*/
217+
public void addDataSourceParser(DataSourceParser dataSourceParser) {
218+
additiveParsers.add(dataSourceParser);
219+
}
220+
221+
private Set<EventSource> loadDataSourceParsers(Diary diary) {
222+
162223
loadDataSourceChannel();
163224
loadJVMEventChannel();
164225
List<DataSourceParser> dataSourceParsers;
165226
if (registeredDataSourceParsers.isEmpty()) {
166227
dataSourceParsers = ServiceLoader.load(DataSourceParser.class)
167228
.stream()
168229
.map(ServiceLoader.Provider::get)
169-
.filter(consumer -> consumer.accepts(diary))
230+
.filter(dataSourceParser -> dataSourceParser.accepts(diary))
170231
.collect(Collectors.toList());
171232
} else{
172233
dataSourceParsers = new ArrayList<>();
@@ -202,18 +263,28 @@ private void loadDataSourceParsers(Diary diary) {
202263
})
203264
.filter(Optional::isPresent)
204265
.map(optional -> (DataSourceParser) optional.get())
205-
.filter(consumer -> consumer.accepts(diary))
266+
.filter(dataSourceParser -> dataSourceParser.accepts(diary))
206267
.collect(Collectors.toList());
207-
if (dataSourceParsers.isEmpty()) {
208-
throw new ServiceConfigurationError("Unable to find a suitable provider to create a DataSourceParser");
209-
}
268+
269+
}
270+
271+
// add in any additional parsers not provided by the module SPI.
272+
dataSourceParsers.addAll(additiveParsers);
273+
274+
if (dataSourceParsers.isEmpty()) {
275+
throw new ServiceConfigurationError("Unable to find a suitable provider to create a DataSourceParser");
210276
}
211277

212278
for (DataSourceParser dataSourceParser : dataSourceParsers) {
279+
LOG_DEBUG_MESSAGE(() -> "Registering " + dataSourceParser.getClass().getName() + " with " + dataSourceChannel.getClass().getName());
213280
dataSourceParser.diary(diary);
214281
dataSourceChannel.registerListener(dataSourceParser);
215282
dataSourceParser.publishTo(jvmEventChannel);
216283
}
284+
285+
return dataSourceParsers.stream()
286+
.map(DataSourceParser::eventsProduced)
287+
.collect(HashSet::new, Set::addAll, Set::addAll);
217288
}
218289

219290
/**
@@ -230,14 +301,57 @@ private void loadDataSourceParsers(Diary diary) {
230301
*/
231302
public JavaVirtualMachine analyze(DataSource<?> dataSource) throws IOException {
232303
GCLogFile logFile = (GCLogFile)dataSource;
233-
loadDataSourceParsers(logFile.diary());
304+
Set<EventSource> events = loadDataSourceParsers(logFile.diary());
234305
JavaVirtualMachine javaVirtualMachine = loadJavaVirtualMachine(logFile);
235306
try {
236-
javaVirtualMachine.analyze(registeredAggregations, jvmEventChannel, dataSourceChannel);
307+
List<Aggregator<? extends Aggregation>> filteredAggregators = filterAggregations(events);
308+
javaVirtualMachine.analyze(filteredAggregators, jvmEventChannel, dataSourceChannel);
237309
} catch(Throwable t) {
238310
LOGGER.log(Level.SEVERE, "Internal Error: Cannot invoke analyze method", t);
239311
}
240312
return javaVirtualMachine;
241313
}
242314

315+
private List<Aggregator<? extends Aggregation>> filterAggregations(Set<EventSource> events) {
316+
List<Aggregator<? extends Aggregation>> aggregators = new ArrayList<>();
317+
for (Aggregation aggregation : registeredAggregations) {
318+
LOG_DEBUG_MESSAGE(() -> "Evaluating: " + aggregation.getClass().getName());
319+
Constructor<? extends Aggregator<?>> constructor = constructor(aggregation);
320+
if (constructor == null) {
321+
LOGGER.log(Level.WARNING, "Cannot find one of: default constructor or @Collates annotation for " + aggregation.getClass().getName());
322+
continue;
323+
}
324+
325+
Aggregator<? extends Aggregation> aggregator = null;
326+
try {
327+
aggregator = constructor.newInstance(aggregation);
328+
} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
329+
LOGGER.log(Level.SEVERE, e.getMessage(), e);
330+
continue;
331+
}
332+
if (events.stream().anyMatch(aggregator::aggregates)) {
333+
LOG_DEBUG_MESSAGE(() -> "Including : " + aggregation.getClass().getName());
334+
aggregators.add(aggregator);
335+
} else {
336+
LOG_DEBUG_MESSAGE(() -> "Excluding : " + aggregation.getClass().getName());
337+
}
338+
}
339+
return aggregators;
340+
341+
}
342+
343+
@SuppressWarnings("unchecked")
344+
private Constructor<? extends Aggregator<?>> constructor(Aggregation aggregation) {
345+
Class<? extends Aggregator<?>> targetClazz = aggregation.collates();
346+
if ( targetClazz != null) {
347+
Constructor<?>[] constructors = targetClazz.getConstructors();
348+
for (Constructor<?> constructor : constructors) {
349+
Parameter[] parameters = constructor.getParameters();
350+
if (parameters.length == 1 && Aggregation.class.isAssignableFrom(parameters[0].getType()))
351+
return (Constructor<? extends Aggregator<?>>) constructor;
352+
}
353+
}
354+
return null;
355+
}
356+
243357
}

‎api/src/main/java/com/microsoft/gctoolkit/aggregator/Aggregator.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,10 +121,8 @@ public void onCompletion(Runnable task) {
121121
* Call a callback when aggregation is completed.
122122
*/
123123
private void complete() {
124-
Runnable t = completionTask;
125-
this.completionTask = null;
126-
if (t != null)
127-
Executors.newSingleThreadExecutor().execute(t);
124+
if (completionTask != null)
125+
Executors.newSingleThreadExecutor().execute(completionTask);
128126

129127
}
130128

‎api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java‎

Lines changed: 9 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License.
33
package com.microsoft.gctoolkit.jvm;
44

5+
import com.microsoft.gctoolkit.GCToolKit;
56
import com.microsoft.gctoolkit.aggregator.Aggregation;
67
import com.microsoft.gctoolkit.aggregator.Aggregator;
78
import com.microsoft.gctoolkit.aggregator.EventSource;
@@ -14,9 +15,6 @@
1415
import com.microsoft.gctoolkit.time.DateTimeStamp;
1516

1617
import java.io.IOException;
17-
import java.lang.reflect.Constructor;
18-
import java.lang.reflect.InvocationTargetException;
19-
import java.lang.reflect.Parameter;
2018
import java.util.List;
2119
import java.util.Map;
2220
import java.util.Optional;
@@ -32,8 +30,6 @@
3230
*/
3331
public abstract class AbstractJavaVirtualMachine implements JavaVirtualMachine {
3432

35-
private boolean debugging = Boolean.getBoolean("microsoft.debug.aggregation");
36-
3733
private static final Logger LOGGER = Logger.getLogger(AbstractJavaVirtualMachine.class.getName());
3834
private static final double LOG_FRAGMENT_THRESHOLD_SECONDS = 60.0d; //todo: replace magic threshold with a heuristic
3935

@@ -152,20 +148,6 @@ public <T extends Aggregation> Optional<T> getAggregation(Class<T> aggregationCl
152148
return Optional.ofNullable((T) aggregatedData.get(aggregationClass));
153149
}
154150

155-
@SuppressWarnings("unchecked")
156-
private Constructor<? extends Aggregator<?>> constructor(Aggregation aggregation) {
157-
Class<? extends Aggregator<?>> targetClazz = aggregation.collates();
158-
if ( targetClazz != null) {
159-
Constructor<?>[] constructors = targetClazz.getConstructors();
160-
for (Constructor<?> constructor : constructors) {
161-
Parameter[] parameters = constructor.getParameters();
162-
if (parameters.length == 1 && Aggregation.class.isAssignableFrom(parameters[0].getType()))
163-
return (Constructor<? extends Aggregator<?>>) constructor;
164-
}
165-
}
166-
return null;
167-
}
168-
169151
/**
170152
* Orchestrate the analysis of a GC log. Step wise
171153
* 1. find the aggregators that aggregate events generated by the gc log
@@ -174,40 +156,24 @@ private Constructor<? extends Aggregator<?>> constructor(Aggregation aggregation
174156
* 4. Wait until all the aggregators have completed
175157
* 5. Set the start and end times
176158
* 6. Return to the caller
177-
* @param registeredAggregations all of the aggregations loaded by the module SPI
159+
* @param registeredAggregators all of the aggregations loaded by the module SPI
178160
* @param eventBus the bus to publish events on
179161
* @param dataSourceBus the bus that raw log lines are published on
180162
*/
181163
@Override
182-
public void analyze(List<Aggregation> registeredAggregations, JVMEventChannel eventBus, DataSourceChannel dataSourceBus) {
164+
public void analyze(List<Aggregator<? extends Aggregation>> registeredAggregators, JVMEventChannel eventBus, DataSourceChannel dataSourceBus) {
183165
Phaser finishLine = new Phaser();
184166
Set<EventSource> generatedEvents = diary.generatesEvents();
185-
for (Aggregation aggregation : registeredAggregations) {
186-
if (debugging)
187-
LOGGER.log(Level.INFO, "Evaluating: " + aggregation.toString());
188-
Constructor<? extends Aggregator<?>> constructor = constructor(aggregation);
189-
if (constructor == null) {
190-
LOGGER.log(Level.WARNING, "Cannot find one of: default constructor or @Collates annotation for " + aggregation.getClass().getName());
191-
continue;
192-
}
193-
if (debugging)
194-
LOGGER.log(Level.INFO, "Loading : " + aggregation.toString());
195-
Aggregator<? extends Aggregation> aggregator = null;
196-
try {
197-
aggregator = constructor.newInstance(aggregation);
198-
} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
199-
LOGGER.log(Level.SEVERE, e.getMessage(), e);
200-
continue;
201-
}
167+
for (Aggregator aggregator : registeredAggregators) {
168+
Aggregation aggregation = aggregator.aggregation();
202169
aggregatedData.put(aggregation.getClass(), aggregation);
203-
Optional<EventSource> source = generatedEvents.stream().filter(aggregator::aggregates).findFirst();
204-
if (source.isPresent()) {
205-
LOGGER.log(Level.FINE, "Registering: " + aggregation.getClass().getName());
170+
generatedEvents.stream().filter(aggregator::aggregates).forEach(eventSource -> {
171+
GCToolKit.LOG_DEBUG_MESSAGE(() -> "Registering " + aggregator.getClass().getName() + " with " + eventSource.toChannel());
206172
finishLine.register();
207173
aggregator.onCompletion(finishLine::arriveAndDeregister);
208-
JVMEventChannelAggregator eventChannelAggregator = new JVMEventChannelAggregator(source.get().toChannel(), aggregator);
174+
JVMEventChannelAggregator eventChannelAggregator = new JVMEventChannelAggregator(eventSource.toChannel(), aggregator);
209175
eventBus.registerListener(eventChannelAggregator);
210-
}
176+
});
211177
}
212178

213179
try {

‎api/src/main/java/com/microsoft/gctoolkit/jvm/JavaVirtualMachine.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.microsoft.gctoolkit.GCToolKit;
77
import com.microsoft.gctoolkit.aggregator.Aggregation;
8+
import com.microsoft.gctoolkit.aggregator.Aggregator;
89
import com.microsoft.gctoolkit.io.DataSource;
910
import com.microsoft.gctoolkit.message.DataSourceChannel;
1011
import com.microsoft.gctoolkit.message.JVMEventChannel;
@@ -122,5 +123,5 @@ public interface JavaVirtualMachine {
122123
* @param eventChannel JVMEvent message channel
123124
* @param dataSourceChannel GC logging data channel
124125
*/
125-
void analyze(List<Aggregation> registeredAggregations, JVMEventChannel eventChannel, DataSourceChannel dataSourceChannel);
126+
void analyze(List<Aggregator<? extends Aggregation>> registeredAggregations, JVMEventChannel eventChannel, DataSourceChannel dataSourceChannel);
126127
}

‎api/src/main/java/com/microsoft/gctoolkit/message/DataSourceParser.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
package com.microsoft.gctoolkit.message;
22

3+
import com.microsoft.gctoolkit.aggregator.EventSource;
34
import com.microsoft.gctoolkit.jvm.Diary;
45

6+
import java.util.Set;
7+
58
public interface DataSourceParser extends DataSourceChannelListener {
69
void publishTo(JVMEventChannel channel);
710
void diary(Diary diary);
811
boolean accepts(Diary diary);
12+
Set<EventSource> eventsProduced();
913
}

‎api/src/test/java/com/microsoft/gctoolkit/io/RotatingGCLogTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package com.microsoft.gctoolkit.io;
2+
3+
import org.junit.jupiter.api.Test;
4+
5+
import java.io.File;
6+
import java.io.IOException;
7+
import java.nio.file.Path;
8+
9+
import static org.junit.jupiter.api.Assertions.assertEquals;
10+
import static org.junit.jupiter.api.Assertions.fail;
11+
12+
public class RotatingGCLogTest {
13+
14+
@Test
15+
void orderRotatingLogsTest() {
16+
Path path = new TestLogFile("G1-80-16gbps2.log").getFile().toPath();
17+
try {
18+
RotatingGCLogFile file = new RotatingGCLogFile(path);
19+
assertEquals(2, file.getMetaData().getNumberOfFiles());
20+
assertEquals(2, file.getMetaData().logFiles().map(LogFileSegment::getPath).map(Path::toFile).map(File::getName).filter(s -> s.startsWith("G1-80-16gbps2")).count());
21+
file.getMetaData().logFiles().map(LogFileSegment::getEndTime).forEach(System.out::println);
22+
} catch (IOException ioe) {
23+
fail(ioe);
24+
}
25+
}
26+
}

0 commit comments

Comments
 (0)