Skip to content
This repository was archived by the owner on May 6, 2022. It is now read-only.

Commit a74b14c

Browse files
author
Florian Lautenschlager
committed
First commit for the chronix importer examples.
1 parent f021105 commit a74b14c

File tree

7 files changed

+515
-0
lines changed

7 files changed

+515
-0
lines changed

‎chronix-importer/build.gradle‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
dependencies {
2+
compile 'org.apache.commons:commons-lang3:3.4'
3+
compile 'commons-io:commons-io:2.5'
4+
5+
compile 'de.qaware.chronix:chronix-api:0.1'
6+
compile 'de.qaware.chronix:chronix-server-client:0.2-beta-1'
7+
compile 'de.qaware.chronix:chronix-kassiopeia-simple:0.1.8-beta-4'
8+
compile 'de.qaware.chronix:chronix-kassiopeia-simple-converter:0.1.8-beta-4'
9+
}

‎chronix-importer/src/main/java/de/qaware/chronix/importer/DatabaseImporter.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package de.qaware.chronix.importer;
2+
3+
import de.qaware.chronix.ChronixClient;
4+
import de.qaware.chronix.converter.KassiopeiaSimpleConverter;
5+
import de.qaware.chronix.converter.serializer.gen.SimpleProtocolBuffers;
6+
import de.qaware.chronix.solr.client.ChronixSolrStorage;
7+
import de.qaware.chronix.timeseries.MetricTimeSeries;
8+
import org.apache.solr.client.solrj.SolrClient;
9+
import org.apache.solr.client.solrj.SolrQuery;
10+
import org.apache.solr.client.solrj.impl.HttpSolrClient;
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
14+
import java.time.Instant;
15+
import java.util.ArrayList;
16+
import java.util.List;
17+
import java.util.function.BiConsumer;
18+
import java.util.stream.Collectors;
19+
20+
/**
21+
* Created by f.lautenschlager on 10.06.2015.
22+
*/
23+
public class DatabaseImporter {
24+
25+
//serialized size of the list
26+
private static final int LIST_SERIALIZED_SIZE = 2;
27+
//serialized size of a point
28+
private static final int POINT_SERIALIZED_SIZE = SimpleProtocolBuffers.Point.newBuilder().setT(Instant.now().toEpochMilli()).setV(4711).build().getSerializedSize();
29+
30+
private static final int SER_SIZE = LIST_SERIALIZED_SIZE + POINT_SERIALIZED_SIZE;
31+
32+
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseImporter.class);
33+
34+
public static ChronixClient<MetricTimeSeries, SolrClient, SolrQuery> CHRONIX = new ChronixClient<>(new KassiopeiaSimpleConverter(), new ChronixSolrStorage<>(200, null, null));
35+
public static final String DATABASE_SERVER_IP = "localhost";
36+
//Solr connection stuff
37+
public static final String SOLR_BASE_URL = "http://" + DATABASE_SERVER_IP + ":8983/solr/";
38+
public static final SolrClient CHRONIX_SOLR_CLIENT = new HttpSolrClient(SOLR_BASE_URL + "chronix");
39+
40+
41+
private DatabaseImporter() {
42+
43+
}
44+
45+
public static BiConsumer<List<ImportPoint>, Metadata> importToChronix() {
46+
47+
return (importPoints, metadata) -> {
48+
49+
50+
LOGGER.info("Chronix ---> Importing {}", metadata);
51+
52+
//String metric = metadata.joinWithoutMetric() + "." + metadata.getMetric();
53+
MetricTimeSeries.Builder builder = new MetricTimeSeries.Builder(metadata.getMetric())
54+
.attribute("host", metadata.getHost())
55+
.attribute("process", metadata.getProcess())
56+
.attribute("group", metadata.getMetricGroup())
57+
.attribute("measurement", metadata.getMeasurement());
58+
59+
//Convert points
60+
importPoints.forEach(point -> builder.point(point.getDate().toEpochMilli(), point.getValue()));
61+
62+
MetricTimeSeries timeSeries = builder.build();
63+
timeSeries.sort();
64+
List<de.qaware.chronix.timeseries.dt.Point> points = timeSeries.points().collect(Collectors.toList());
65+
66+
final int chunkSize = 128 * 1024;
67+
68+
//number of points
69+
int numberOfPoints = chunkSize / SER_SIZE;
70+
int start = 0;
71+
int end;
72+
73+
List<MetricTimeSeries> splits = new ArrayList<>();
74+
//Loop over the time series
75+
while (start <= timeSeries.size()) {
76+
77+
if (timeSeries.size() - (start + numberOfPoints) > 0) {
78+
end = start + numberOfPoints;
79+
} else {
80+
end = timeSeries.size();
81+
}
82+
83+
List<de.qaware.chronix.timeseries.dt.Point> sublist = points.subList(start, end);
84+
start += numberOfPoints;
85+
86+
MetricTimeSeries.Builder toAddTs = new MetricTimeSeries.Builder(metadata.getMetric())
87+
.attribute("host", metadata.getHost())
88+
.attribute("process", metadata.getProcess())
89+
.attribute("group", metadata.getMetricGroup());
90+
toAddTs.attributes(timeSeries.attributes());
91+
92+
sublist.forEach(pair -> toAddTs.point(pair.getTimestamp(), pair.getValue()));
93+
94+
splits.add(toAddTs.build());
95+
}
96+
97+
CHRONIX.add(splits, CHRONIX_SOLR_CLIENT);
98+
};
99+
100+
}
101+
}

‎chronix-importer/src/main/java/de/qaware/chronix/importer/FileImporter.java‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
package de.qaware.chronix.importer;
2+
3+
import org.apache.commons.io.FileUtils;
4+
import org.apache.commons.io.IOUtils;
5+
import org.apache.commons.lang3.StringUtils;
6+
import org.slf4j.Logger;
7+
import org.slf4j.LoggerFactory;
8+
9+
import java.io.*;
10+
import java.text.DecimalFormat;
11+
import java.text.Normalizer;
12+
import java.text.NumberFormat;
13+
import java.text.SimpleDateFormat;
14+
import java.time.Instant;
15+
import java.util.*;
16+
import java.util.concurrent.atomic.AtomicInteger;
17+
import java.util.function.BiConsumer;
18+
import java.util.zip.GZIPInputStream;
19+
20+
/**
21+
* Created by f.lautenschlager on 10.06.2015.
22+
*/
23+
public class FileImporter {
24+
25+
private static final Logger LOGGER = LoggerFactory.getLogger(FileImporter.class);
26+
27+
private static final String METRICS_FILE_PATH = "metrics.csv";
28+
29+
private FileImporter() {
30+
31+
}
32+
33+
@SafeVarargs
34+
public static final Pair<Integer, Integer> importPoints(Map<Metadata, Pair<Instant, Instant>> points, File folder, BiConsumer<List<ImportPoint>, Metadata>... databases) {
35+
36+
final AtomicInteger pointCounter = new AtomicInteger(0);
37+
final AtomicInteger tsCounter = new AtomicInteger(0);
38+
final File metricsFile = new File(METRICS_FILE_PATH);
39+
try {
40+
final FileWriter metricsFileWriter = new FileWriter(metricsFile);
41+
42+
43+
Collection<File> files = new ArrayList<>();
44+
if (folder.isFile()) {
45+
files.add(folder);
46+
} else {
47+
files.addAll(FileUtils.listFiles(folder, new String[]{"gz", "csv"}, true));
48+
}
49+
50+
AtomicInteger counter = new AtomicInteger(0);
51+
52+
files.parallelStream().forEach(file -> {
53+
boolean onlyMinusOne = true;
54+
SimpleDateFormat sdf = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss.SSS");
55+
56+
57+
NumberFormat nf = DecimalFormat.getInstance(Locale.ENGLISH);
58+
InputStream inputStream = null;
59+
BufferedReader reader = null;
60+
try {
61+
inputStream = new FileInputStream(file);
62+
63+
if (file.getName().endsWith("gz")) {
64+
inputStream = new GZIPInputStream(inputStream);
65+
}
66+
reader = new BufferedReader(new InputStreamReader(inputStream));
67+
68+
//Read the first line
69+
String headerLine = reader.readLine();
70+
71+
if (headerLine == null || headerLine.isEmpty()) {
72+
boolean deleted = deleteFile(file, inputStream, reader);
73+
LOGGER.debug("File is empty {}. File {} removed {}", file.getName(), deleted);
74+
return;
75+
}
76+
77+
78+
//host _ process _ metricGroup
79+
String[] fileNameMetaData = file.getName().split("_");
80+
//build meta data object
81+
String host = fileNameMetaData[0];
82+
String process = fileNameMetaData[1];
83+
String metricGroup = fileNameMetaData[2];
84+
String measurement = file.getParentFile().getName();
85+
86+
87+
String[] metrics = headerLine.split(";");
88+
89+
Map<Integer, Metadata> metadatas = new HashMap<>(metrics.length);
90+
91+
92+
for (int i = 1; i < metrics.length; i++) {
93+
String metric = metrics[i];
94+
String metricOnlyAscii = Normalizer.normalize(metric, Normalizer.Form.NFD);
95+
metricOnlyAscii = metric.replaceAll("[^\\x00-\\x7F]", "");
96+
Metadata metadata = new Metadata(host, process, metricGroup, metricOnlyAscii, measurement);
97+
98+
//Check if meta data is completely set
99+
if (isEmpty(metadata)) {
100+
boolean deleted = deleteFile(file, inputStream, reader);
101+
LOGGER.info("Metadata contains empty values {}. File {} deleted {}", metadata, file.getName(), deleted);
102+
continue;
103+
}
104+
105+
if (metadata.getMetric().equals(".*")) {
106+
boolean deleted = deleteFile(file, inputStream, reader);
107+
LOGGER.info("Metadata metric{}. File {} deleted {}", metadata.getMetric(), file.getName(), deleted);
108+
continue;
109+
}
110+
metadatas.put(i, metadata);
111+
tsCounter.incrementAndGet();
112+
113+
}
114+
115+
Map<Integer, List<ImportPoint>> dataPoints = new HashMap<>();
116+
117+
String line;
118+
boolean instantdate = true;
119+
boolean onlyOnce = true;
120+
while ((line = reader.readLine()) != null) {
121+
String[] splits = line.split(";");
122+
String date = splits[0];
123+
124+
if (onlyOnce) {
125+
try {
126+
Instant.parse(date);
127+
} catch (Exception e) {
128+
instantdate = false;
129+
}
130+
onlyOnce = false;
131+
}
132+
Instant dateObject;
133+
if (instantdate) {
134+
dateObject = Instant.parse(date);
135+
} else {
136+
dateObject = sdf.parse(date).toInstant();
137+
}
138+
139+
140+
String[] values = splits;
141+
142+
for (int column = 1; column < values.length; column++) {
143+
144+
String value = values[column];
145+
double numericValue = nf.parse(value).doubleValue();
146+
147+
ImportPoint point;
148+
if (instantdate) {
149+
point = new ImportPoint(dateObject, numericValue);
150+
} else {
151+
point = new ImportPoint(dateObject, numericValue);
152+
153+
}
154+
155+
if (!dataPoints.containsKey(column)) {
156+
dataPoints.put(column, new ArrayList<>());
157+
}
158+
dataPoints.get(column).add(point);
159+
pointCounter.incrementAndGet();
160+
}
161+
162+
}
163+
/*
164+
if (onlyMinusOne) {
165+
pointCounter.addAndGet(-dataPoints.size());
166+
tsCounter.decrementAndGet();
167+
168+
//close all streams
169+
boolean deleted = deleteFile(file, inputStream, reader);
170+
LOGGER.info("{} contains only -1. Deleted {}", file.getName(), deleted);
171+
172+
return;
173+
}
174+
*/
175+
176+
dataPoints.values().forEach(Collections::sort);
177+
178+
IOUtils.closeQuietly(reader);
179+
IOUtils.closeQuietly(inputStream);
180+
181+
dataPoints.forEach((key, value) -> {
182+
for (BiConsumer<List<ImportPoint>, Metadata> database : databases) {
183+
database.accept(value, metadatas.get(key));
184+
}
185+
points.put(metadatas.get(key), Pair.of(value.get(0).getDate(), value.get(value.size() - 1).getDate()));
186+
//write the stats to the file
187+
Instant start = value.get(0).getDate();
188+
Instant end = value.get(value.size() - 1).getDate();
189+
190+
try {
191+
writeStatsLine(metricsFileWriter, metadatas.get(key), start, end);
192+
} catch (IOException e) {
193+
LOGGER.error("Could not write stats line", e);
194+
}
195+
LOGGER.info("{} of {} time series imported", counter.incrementAndGet(), tsCounter.get());
196+
});
197+
198+
199+
} catch (Exception e) {
200+
LOGGER.info("Exception while reading points.", e);
201+
} finally {
202+
//close all streams
203+
IOUtils.closeQuietly(reader);
204+
IOUtils.closeQuietly(inputStream);
205+
}
206+
207+
});
208+
} catch (Exception e) {
209+
LOGGER.error("Exception occurred during reading points.");
210+
}
211+
return Pair.of(tsCounter.get(), pointCounter.get());
212+
}
213+
214+
private static void writeStatsLine(FileWriter metricsFile, Metadata metadata, Instant start, Instant end) throws IOException {
215+
//host:process:metric-group:metric:start:end
216+
StringBuilder line = new StringBuilder();
217+
line.append(metadata.getHost()).append(";")
218+
.append(metadata.getProcess()).append(";")
219+
.append(metadata.getMetricGroup()).append(";")
220+
.append(metadata.getMetric()).append(";")
221+
.append(start).append(";")
222+
.append(end).append(";")
223+
.append("\n");
224+
225+
metricsFile.write(line.toString());
226+
metricsFile.flush();
227+
228+
}
229+
230+
private static boolean deleteFile(File file, InputStream inputStream, BufferedReader reader) {
231+
IOUtils.closeQuietly(reader);
232+
IOUtils.closeQuietly(inputStream);
233+
234+
//remove file
235+
return file.delete();
236+
}
237+
238+
private static boolean isEmpty(Metadata metadata) {
239+
return empty(metadata.getMetric()) || empty(metadata.getMetricGroup()) || empty(metadata.getProcess()) || empty(metadata.getHost());
240+
}
241+
242+
private static boolean empty(String metric) {
243+
return StringUtils.isEmpty(metric);
244+
}
245+
}

‎chronix-importer/src/main/java/de/qaware/chronix/importer/ImportPoint.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package de.qaware.chronix.importer;
2+
3+
import java.time.Instant;
4+
5+
/**
6+
* Created by f.lautenschlager on 10.06.2015.
7+
*/
8+
public class ImportPoint implements Comparable<ImportPoint>{
9+
10+
private Instant date;
11+
private double value;
12+
13+
public ImportPoint(Instant date, Double value) {
14+
this.date = date;
15+
this.value = value;
16+
}
17+
18+
public Instant getDate() {
19+
return date;
20+
}
21+
22+
public double getValue() {
23+
return value;
24+
}
25+
26+
@Override
27+
public int compareTo(ImportPoint o) {
28+
return this.date.compareTo(o.date);
29+
}
30+
31+
@Override
32+
public String toString() {
33+
return "ImportPoint{" +
34+
"date=" + date +
35+
", value=" + value +
36+
'}';
37+
}
38+
}

0 commit comments

Comments
 (0)