264 lines
8.9 KiB
Rust
264 lines
8.9 KiB
Rust
use crate::metrics;
|
|
use crate::types::BenchmarkResult;
|
|
|
|
fn format_rss_mb(kb: u64) -> String {
|
|
format!("{:.1}MB", kb as f64 / 1024.0)
|
|
}
|
|
|
|
fn format_faults(count: u64) -> String {
|
|
if count < 1000 {
|
|
count.to_string()
|
|
} else {
|
|
let s = count.to_string();
|
|
let mut result = String::new();
|
|
for (i, c) in s.chars().rev().enumerate() {
|
|
if i > 0 && i % 3 == 0 {
|
|
result.push(',');
|
|
}
|
|
result.push(c);
|
|
}
|
|
result.chars().rev().collect()
|
|
}
|
|
}
|
|
|
|
// Format benchmark results as Markdown, grouped by category.
|
|
pub fn format_report(results: &[BenchmarkResult]) -> String {
|
|
let mut report = String::new();
|
|
report.push_str("# Benchmark: mmap vs pread\n\n");
|
|
|
|
let mut categories: std::collections::BTreeMap<&str, Vec<&BenchmarkResult>> =
|
|
std::collections::BTreeMap::new();
|
|
for r in results {
|
|
categories.entry(&r.category).or_default().push(r);
|
|
}
|
|
|
|
for (category, category_results) in &categories {
|
|
report.push_str(&format!("## {}\n\n", capitalize(category)));
|
|
|
|
let mut tests: std::collections::BTreeMap<&str, Vec<&BenchmarkResult>> =
|
|
std::collections::BTreeMap::new();
|
|
for r in category_results {
|
|
tests.entry(&r.test_name).or_default().push(r);
|
|
}
|
|
|
|
let mut variants: Vec<(String, String)> = Vec::new();
|
|
for r in category_results {
|
|
if !variants
|
|
.iter()
|
|
.any(|(b, v)| b == &r.backend && v == &r.variant)
|
|
{
|
|
variants.push((r.backend.clone(), r.variant.clone()));
|
|
}
|
|
}
|
|
variants.sort();
|
|
|
|
report.push_str("### Latency\n\n");
|
|
report.push_str("| Test |");
|
|
for (backend, variant) in &variants {
|
|
report.push_str(&format!(" {} ({}) |", backend, variant));
|
|
}
|
|
report.push_str(" Winner |\n");
|
|
|
|
report.push_str("|------|");
|
|
for _ in &variants {
|
|
report.push_str("------|");
|
|
}
|
|
report.push_str("--------|\n");
|
|
|
|
for (test_name, test_results) in &tests {
|
|
report.push_str(&format!("| {} |", test_name));
|
|
|
|
let mut all_means: Vec<f64> = Vec::new();
|
|
for (backend, variant) in &variants {
|
|
if let Some(r) = test_results
|
|
.iter()
|
|
.find(|r| r.backend == *backend && r.variant == *variant)
|
|
{
|
|
let avg = if r.latency_us.is_empty() {
|
|
0.0
|
|
} else {
|
|
metrics::mean(&r.latency_us)
|
|
};
|
|
all_means.push(avg);
|
|
}
|
|
}
|
|
let use_ms = all_means.iter().any(|&v| v >= 1000.0);
|
|
|
|
let mut best_backend = String::new();
|
|
let mut best_latency = f64::MAX;
|
|
|
|
for (backend, variant) in &variants {
|
|
let matching = test_results
|
|
.iter()
|
|
.find(|r| r.backend == *backend && r.variant == *variant);
|
|
if let Some(r) = matching {
|
|
let avg = if r.latency_us.is_empty() {
|
|
0.0
|
|
} else {
|
|
metrics::mean(&r.latency_us)
|
|
};
|
|
let sd = metrics::stdev(&r.latency_us);
|
|
let p95_val = metrics::p95(&r.latency_us) as f64;
|
|
|
|
let cell = if use_ms {
|
|
format!(
|
|
"{:.2}\u{00b1}{:.2}ms (p95:{:.2}ms)",
|
|
avg / 1000.0,
|
|
sd / 1000.0,
|
|
p95_val / 1000.0
|
|
)
|
|
} else {
|
|
format!(
|
|
"{:.1}\u{00b1}{:.1}\u{b5}s (p95:{:.1}\u{b5}s)",
|
|
avg, sd, p95_val
|
|
)
|
|
};
|
|
report.push_str(&format!(" {} |", cell));
|
|
if avg > 0.0 && avg < best_latency {
|
|
best_latency = avg;
|
|
best_backend = format!("{} ({})", backend, variant);
|
|
}
|
|
} else {
|
|
report.push_str(" - |");
|
|
}
|
|
}
|
|
report.push_str(&format!(
|
|
" {} |\n",
|
|
if best_backend.is_empty() {
|
|
"-"
|
|
} else {
|
|
&best_backend
|
|
}
|
|
));
|
|
}
|
|
|
|
report.push('\n');
|
|
|
|
let has_memory = category_results
|
|
.iter()
|
|
.any(|r| r.rss_kb > 0 || r.rss_peak_kb > 0);
|
|
|
|
if has_memory {
|
|
report.push_str("### Memory\n\n");
|
|
report.push_str("| Test | Variant | RSS | Peak RSS | Page Faults |\n");
|
|
report.push_str("|------|---------|-----|----------|-------------|\n");
|
|
|
|
let mut mem_rows: Vec<&BenchmarkResult> = category_results.to_vec();
|
|
mem_rows.sort_by(|a, b| {
|
|
(&a.test_name, &a.backend, &a.variant)
|
|
.cmp(&(&b.test_name, &b.backend, &b.variant))
|
|
});
|
|
for r in mem_rows {
|
|
let variant_label = format!("{} ({})", r.backend, r.variant);
|
|
report.push_str(&format!(
|
|
"| {} | {} | {} | {} | {} |\n",
|
|
r.test_name,
|
|
variant_label,
|
|
format_rss_mb(r.rss_kb),
|
|
format_rss_mb(r.rss_peak_kb),
|
|
format_faults(r.page_faults),
|
|
));
|
|
}
|
|
report.push('\n');
|
|
}
|
|
|
|
type ExtraEntry = (String, f64);
|
|
type ExtraGroup = (String, String, Vec<ExtraEntry>);
|
|
|
|
let mut extras: Vec<ExtraGroup> = category_results
|
|
.iter()
|
|
.filter(|r| !r.extra.is_empty())
|
|
.map(|r| {
|
|
let mut pairs: Vec<(String, f64)> =
|
|
r.extra.iter().map(|(k, &v)| (k.clone(), v)).collect();
|
|
pairs.sort_by(|a, b| a.0.cmp(&b.0));
|
|
(
|
|
r.test_name.clone(),
|
|
format!("{} ({})", r.backend, r.variant),
|
|
pairs,
|
|
)
|
|
})
|
|
.collect();
|
|
extras.sort_by(|a, b| (&a.0, &a.1).cmp(&(&b.0, &b.1)));
|
|
|
|
if !extras.is_empty() {
|
|
report.push_str("### Extra Metrics\n\n");
|
|
report.push_str("| Test | Variant | Metric | Value |\n");
|
|
report.push_str("|------|---------|--------|-------|\n");
|
|
|
|
for (test_name, variant_label, pairs) in &extras {
|
|
for (key, val) in pairs {
|
|
report.push_str(&format!(
|
|
"| {} | {} | {} | {:.3} |\n",
|
|
test_name, variant_label, key, val
|
|
));
|
|
}
|
|
}
|
|
report.push('\n');
|
|
}
|
|
}
|
|
|
|
report.push_str("## Summary\n\n");
|
|
report.push_str(&format!("- Total benchmarks: {}\n", results.len()));
|
|
report.push_str("- Categories: ");
|
|
report.push_str(&categories.keys().cloned().collect::<Vec<_>>().join(", "));
|
|
report.push('\n');
|
|
|
|
report
|
|
}
|
|
|
|
fn capitalize(s: &str) -> String {
|
|
let mut c = s.chars();
|
|
match c.next() {
|
|
None => String::new(),
|
|
Some(f) => f.to_uppercase().chain(c).collect(),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::collections::HashMap;
|
|
|
|
fn make_result(
|
|
category: &str,
|
|
test_name: &str,
|
|
backend: &str,
|
|
variant: &str,
|
|
latency_us: Vec<u64>,
|
|
) -> BenchmarkResult {
|
|
BenchmarkResult {
|
|
category: category.to_string(),
|
|
test_name: test_name.to_string(),
|
|
backend: backend.to_string(),
|
|
variant: variant.to_string(),
|
|
latency_us,
|
|
rss_kb: 0,
|
|
rss_peak_kb: 0,
|
|
page_faults: 0,
|
|
extra: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn report_ordering_independent_of_input_order() {
|
|
let set_a = vec![
|
|
make_result("sequential", "read_1mb", "pread", "default", vec![100, 110, 105]),
|
|
make_result("sequential", "read_1mb", "mmap", "default", vec![80, 85, 90]),
|
|
make_result("sequential", "read_4kb", "pread", "default", vec![10, 12, 11]),
|
|
make_result("sequential", "read_4kb", "mmap", "default", vec![8, 9, 7]),
|
|
];
|
|
|
|
let set_b = vec![
|
|
make_result("sequential", "read_4kb", "mmap", "default", vec![8, 9, 7]),
|
|
make_result("sequential", "read_1mb", "mmap", "default", vec![80, 85, 90]),
|
|
make_result("sequential", "read_4kb", "pread", "default", vec![10, 12, 11]),
|
|
make_result("sequential", "read_1mb", "pread", "default", vec![100, 110, 105]),
|
|
];
|
|
|
|
let report_a = format_report(&set_a);
|
|
let report_b = format_report(&set_b);
|
|
assert_eq!(report_a, report_b, "Reports must be identical regardless of input order");
|
|
}
|
|
}
|