fix(bench): wrap file writes with BufWriter to reduce syscall overhead
Add BufWriter::with_capacity(64KB) to generate_test_file, generate_growable_file, and append_lines in data_gen.rs. Previously each writeln! triggered an individual write syscall, making 5GB/74M-line benchmark data generation extremely slow. BufWriter batches writes into 64KB chunks, reducing syscalls by ~1000x. Explicit flush()? + drop before subsequent reads ensures data visibility and propagates flush errors (BufWriter::drop swallows them). Closes #35
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
use std::fs;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::io::{BufRead, BufReader, BufWriter, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct TestFileInfo {
|
||||
@@ -51,7 +51,7 @@ fn generate_test_file(path: &Path) -> std::io::Result<TestFileInfo> {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let mut file = fs::File::create(path)?;
|
||||
let mut file = BufWriter::with_capacity(64 * 1024, fs::File::create(path)?);
|
||||
let target_lines: u64 = 74_000_000;
|
||||
for i in 0..target_lines {
|
||||
writeln!(
|
||||
@@ -63,6 +63,8 @@ fn generate_test_file(path: &Path) -> std::io::Result<TestFileInfo> {
|
||||
i * 7
|
||||
)?;
|
||||
}
|
||||
file.flush()?;
|
||||
drop(file);
|
||||
|
||||
get_file_info(path)
|
||||
}
|
||||
@@ -72,7 +74,7 @@ pub fn generate_growable_file(dir: &Path) -> std::io::Result<PathBuf> {
|
||||
fs::create_dir_all(dir)?;
|
||||
let path = dir.join("growable.log");
|
||||
|
||||
let mut file = fs::File::create(&path)?;
|
||||
let mut file = BufWriter::with_capacity(64 * 1024, fs::File::create(&path)?);
|
||||
for i in 0..150_000u64 {
|
||||
writeln!(
|
||||
file,
|
||||
@@ -82,13 +84,17 @@ pub fn generate_growable_file(dir: &Path) -> std::io::Result<PathBuf> {
|
||||
i
|
||||
)?;
|
||||
}
|
||||
file.flush()?;
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// Append `count` lines to the file
|
||||
pub fn append_lines(path: &Path, count: usize) -> std::io::Result<()> {
|
||||
let mut file = fs::OpenOptions::new().append(true).open(path)?;
|
||||
let mut file = BufWriter::with_capacity(
|
||||
64 * 1024,
|
||||
fs::OpenOptions::new().append(true).open(path)?,
|
||||
);
|
||||
let existing_lines = count_existing_lines(path).unwrap_or(0);
|
||||
for i in 0..count {
|
||||
writeln!(
|
||||
@@ -97,6 +103,7 @@ pub fn append_lines(path: &Path, count: usize) -> std::io::Result<()> {
|
||||
existing_lines + i as u64
|
||||
)?;
|
||||
}
|
||||
file.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user