Skip to content

Commit 2803d5d

Browse files
spoutn1kspoutn1k
committed
Add memory-benchmarking
1 parent 695e858 commit 2803d5d

File tree

6 files changed

+200
-187
lines changed

6 files changed

+200
-187
lines changed

‎Cargo.lock‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[workspace]
2-
members = ['lustre-collector', 'lustrefs-exporter']
2+
members = ['lustre-collector', 'lustrefs-exporter', "memory-benchmarking"]
33
resolver = "3"
44

55
[workspace.package]

‎lustre-collector/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ version.workspace = true
99
[dependencies]
1010
clap = { workspace = true, features = ["derive"] }
1111
combine.workspace = true
12+
memory-benchmarking = { version = "0.11.0", path = "../memory-benchmarking" }
1213
serde = { workspace = true, features = ["derive"] }
1314
serde_json.workspace = true
1415
serde_yaml.workspace = true

‎lustre-collector/benches/combine_memory.rs‎

Lines changed: 1 addition & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -1,136 +1,8 @@
11
use combine::parser::EasyParser;
22
use criterion::{Criterion, criterion_group, criterion_main};
33
use lustre_collector::quota::parse as combine_parse;
4+
use memory_benchmarking::{BencherOutput, MemoryUsage, aggregate_samples, get_memory_stats};
45
use std::{fs::File, io::Read, time::Duration};
5-
use sysinfo::{Pid, ProcessExt, System, SystemExt};
6-
7-
#[derive(serde::Serialize)]
8-
struct BencherOutput {
9-
memory_usage: BencherMetrics,
10-
}
11-
12-
#[derive(serde::Serialize)]
13-
struct BencherMetrics {
14-
start_rss_mib: MetricEntry,
15-
peak_rss_mib: MetricEntry,
16-
end_rss_mib: MetricEntry,
17-
memory_growth_mib: MetricEntry,
18-
peak_over_start_rss_ratio: MetricEntry,
19-
avg_runtime_rss_mib: MetricEntry,
20-
start_virtual_mib: MetricEntry,
21-
peak_virtual_mib: MetricEntry,
22-
end_virtual_mib: MetricEntry,
23-
virtual_growth_mib: MetricEntry,
24-
peak_over_start_virtual_ratio: MetricEntry,
25-
avg_runtime_virtual_mib: MetricEntry,
26-
}
27-
28-
#[derive(serde::Serialize)]
29-
struct MetricEntry {
30-
value: f64,
31-
#[serde(skip_serializing_if = "Option::is_none")]
32-
lower_value: Option<f64>,
33-
#[serde(skip_serializing_if = "Option::is_none")]
34-
upper_value: Option<f64>,
35-
}
36-
37-
#[derive(Clone, Debug, serde::Serialize)]
38-
struct MemoryUsage {
39-
start_rss: f64,
40-
end_rss: f64,
41-
memory_growth: f64,
42-
peak_over_start_rss_ratio: f64,
43-
avg_rss: f64,
44-
min_rss: f64,
45-
max_rss: f64,
46-
start_virtual: f64,
47-
end_virtual: f64,
48-
virtual_growth: f64,
49-
peak_over_start_virtual_ratio: f64,
50-
avg_virtual: f64,
51-
min_virtual: f64,
52-
max_virtual: f64,
53-
}
54-
55-
impl From<MemoryUsage> for BencherOutput {
56-
fn from(x: MemoryUsage) -> Self {
57-
BencherOutput {
58-
memory_usage: BencherMetrics {
59-
start_rss_mib: MetricEntry {
60-
value: x.start_rss,
61-
lower_value: None,
62-
upper_value: None,
63-
},
64-
peak_rss_mib: MetricEntry {
65-
value: x.max_rss,
66-
lower_value: None,
67-
upper_value: None,
68-
},
69-
end_rss_mib: MetricEntry {
70-
value: x.end_rss,
71-
lower_value: None,
72-
upper_value: None,
73-
},
74-
memory_growth_mib: MetricEntry {
75-
value: x.memory_growth,
76-
lower_value: None,
77-
upper_value: None,
78-
},
79-
peak_over_start_rss_ratio: MetricEntry {
80-
value: x.peak_over_start_rss_ratio,
81-
lower_value: None,
82-
upper_value: None,
83-
},
84-
avg_runtime_rss_mib: MetricEntry {
85-
value: x.avg_rss,
86-
lower_value: Some(x.min_rss),
87-
upper_value: Some(x.max_rss),
88-
},
89-
start_virtual_mib: MetricEntry {
90-
value: x.start_virtual,
91-
lower_value: None,
92-
upper_value: None,
93-
},
94-
peak_virtual_mib: MetricEntry {
95-
value: x.max_virtual,
96-
lower_value: None,
97-
upper_value: None,
98-
},
99-
end_virtual_mib: MetricEntry {
100-
value: x.end_virtual,
101-
lower_value: None,
102-
upper_value: None,
103-
},
104-
virtual_growth_mib: MetricEntry {
105-
value: x.virtual_growth,
106-
lower_value: None,
107-
upper_value: None,
108-
},
109-
peak_over_start_virtual_ratio: MetricEntry {
110-
value: x.peak_over_start_virtual_ratio,
111-
lower_value: None,
112-
upper_value: None,
113-
},
114-
avg_runtime_virtual_mib: MetricEntry {
115-
value: x.avg_virtual,
116-
lower_value: Some(x.min_virtual),
117-
upper_value: Some(x.max_virtual),
118-
},
119-
},
120-
}
121-
}
122-
}
123-
124-
fn get_memory_stats() -> (f64, f64) {
125-
let mut system = System::new();
126-
system.refresh_process(Pid::from(std::process::id() as usize));
127-
128-
if let Some(process) = system.process(Pid::from(std::process::id() as usize)) {
129-
(process.memory() as f64, process.virtual_memory() as f64)
130-
} else {
131-
(0.0, 0.0)
132-
}
133-
}
1346

1357
async fn test_combine_with_mem(buffer: &str) -> MemoryUsage {
1368
let (start_rss, start_virtual) = get_memory_stats();
@@ -260,62 +132,5 @@ pub fn combine_memory(c: &mut Criterion) {
260132
}
261133
}
262134

263-
fn aggregate_samples(samples: &[MemoryUsage]) -> MemoryUsage {
264-
let size = samples.len() as f64;
265-
const MIB_CONVERSION_FACTOR: f64 = 1_048_576.0;
266-
267-
MemoryUsage {
268-
start_rss: samples.iter().map(|x| x.start_rss).sum::<f64>() / size / MIB_CONVERSION_FACTOR,
269-
end_rss: samples.iter().map(|x| x.end_rss).sum::<f64>() / size / MIB_CONVERSION_FACTOR,
270-
memory_growth: samples.iter().map(|x| x.memory_growth).sum::<f64>()
271-
/ size
272-
/ MIB_CONVERSION_FACTOR,
273-
peak_over_start_rss_ratio: samples
274-
.iter()
275-
.map(|x| x.peak_over_start_rss_ratio)
276-
.sum::<f64>()
277-
/ size,
278-
avg_rss: samples.iter().map(|x| x.avg_rss).sum::<f64>() / size / MIB_CONVERSION_FACTOR,
279-
min_rss: samples
280-
.iter()
281-
.map(|x| x.min_rss)
282-
.fold(f64::INFINITY, f64::min)
283-
/ MIB_CONVERSION_FACTOR,
284-
max_rss: samples
285-
.iter()
286-
.map(|x| x.max_rss)
287-
.fold(f64::NEG_INFINITY, f64::max)
288-
/ MIB_CONVERSION_FACTOR,
289-
290-
start_virtual: samples.iter().map(|x| x.start_virtual).sum::<f64>()
291-
/ size
292-
/ MIB_CONVERSION_FACTOR,
293-
end_virtual: samples.iter().map(|x| x.end_virtual).sum::<f64>()
294-
/ size
295-
/ MIB_CONVERSION_FACTOR,
296-
virtual_growth: samples.iter().map(|x| x.virtual_growth).sum::<f64>()
297-
/ size
298-
/ MIB_CONVERSION_FACTOR,
299-
peak_over_start_virtual_ratio: samples
300-
.iter()
301-
.map(|x| x.peak_over_start_virtual_ratio)
302-
.sum::<f64>()
303-
/ size,
304-
avg_virtual: samples.iter().map(|x| x.avg_virtual).sum::<f64>()
305-
/ size
306-
/ MIB_CONVERSION_FACTOR,
307-
min_virtual: samples
308-
.iter()
309-
.map(|x| x.min_virtual)
310-
.fold(f64::INFINITY, f64::min)
311-
/ MIB_CONVERSION_FACTOR,
312-
max_virtual: samples
313-
.iter()
314-
.map(|x| x.max_virtual)
315-
.fold(f64::NEG_INFINITY, f64::max)
316-
/ MIB_CONVERSION_FACTOR,
317-
}
318-
}
319-
320135
criterion_group!(benches, combine_memory);
321136
criterion_main!(benches);

‎memory-benchmarking/Cargo.toml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[package]
2+
name = "memory-benchmarking"
3+
authors.workspace = true
4+
edition.workspace = true
5+
license.workspace = true
6+
version.workspace = true
7+
8+
[dependencies]
9+
serde = { workspace = true, features = ["derive"] }
10+
sysinfo.workspace = true
11+
12+
[lints]
13+
workspace = true

0 commit comments

Comments
 (0)