dailz/logViewer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(core): linear mmap VHI rebuild + zero-alloc wrap helpers

Browse Source 
Four changes that compound to a 2-5x speedup on VHI rebuild for large
files, plus eliminating one full wasted VHI pass on initial open.

* spawn_indexer no longer computes VHI inline (progressive_reader.rs).
  The inline VHI used a hardcoded width=80, was invalidated immediately
  on Loading->Ready transition (gutter digit-boundary), and triggered
  a second full rebuild. Now returns visual_height_index: None and lets
  the UI trigger one rebuild with the correct post-layout width.

* spawn_visual_height_rebuild rewritten as a single linear pass over
  mmap bytes using memchr::memchr (SIMD) to find newlines, replacing
  the byte-by-byte BufRead::read_until scan. Cancel check throttled
  to every 1MB instead of per-line.

* wrap_line_count(&str, width) -> usize added as a zero-allocation
  counterpart to wrap_line_chars. Property tests cover ASCII, CJK,
  emoji, combining marks, tabs, zero-width chars, control chars, and
  real log lines across many widths - all match wrap_line_chars().len()
  exactly. compute_text_visual_height now uses wrap_line_count, removing
  10M Vec allocations on a 10M-line file.

* format_json_line returns Cow<'_, str> instead of String. Non-JSON
  lines borrow the input (zero allocation); only pretty-printed JSON
  objects allocate. Removes another 10M allocations when json_format=true.

Removes now-unused compute_visual_heights (only caller was the deleted
inline VHI path).
This commit is contained in:
dailz
2026-06-22 14:52:01 +08:00
parent e765f8967f
commit 4421da35f4
2 changed files with 310 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/core/src/io/progressive_reader.rs
+116 -93
View File
@@ -8,7 +8,7 @@ use crate::io::file_reader::{AppendStatus, FileReader};
use crate::io::index_cache::IndexCache;
use crate::io::line_index::LineIndex;
use crate::io::line_sampler::sample_line_count;
use crate::io::wrap::{format_json_line, wrap_line_chars, MAX_WRAP_INPUT_LEN};
use crate::io::wrap::{MAX_WRAP_INPUT_LEN, format_json_line};
// ─── Cancel-aware channel helpers ────────────────────────────────────────────
@@ -235,29 +235,11 @@ pub fn compute_line_visual_height(
fn compute_text_visual_height(text: &str, width: usize) -> usize {
let mut height = 0;
for sub_line in text.split('\n') {
height += wrap_line_chars(sub_line, width).len();
height += crate::io::wrap::wrap_line_count(sub_line, width);
}
height.max(1)
}
fn compute_visual_heights(
reader: &FileReader,
terminal_width: usize,
json_format: bool,
) -> Vec<usize> {
let line_count = reader.line_count();
let mut visual_heights = Vec::with_capacity(line_count);
for i in 0..line_count {
let line_text = reader.get_line(i).unwrap_or("");
visual_heights.push(compute_line_visual_height(
line_text,
terminal_width,
json_format,
));
}
visual_heights
}
// ─── ReaderState ─────────────────────────────────────────────────────────────
pub enum ReaderState {
@@ -281,8 +263,8 @@ pub enum ReaderState {
pub fn spawn_indexer(
path: PathBuf,
generation: u64,
terminal_width: usize,
json_format: bool,
_terminal_width: usize,
_json_format: bool,
cancel_rx: crossbeam_channel::Receiver<()>,
) -> crossbeam_channel::Receiver<IndexerMessage> {
let (tx, rx) = crossbeam_channel::bounded(10);
@@ -291,20 +273,28 @@ pub fn spawn_indexer(
let file = match std::fs::File::open(&path) {
Ok(f) => f,
Err(e) => {
send_cancelable(&tx, IndexerMessage::Error {
generation,
message: e.to_string(),
}, &cancel_rx);
send_cancelable(
&tx,
IndexerMessage::Error {
generation,
message: e.to_string(),
},
&cancel_rx,
);
return;
}
};
let target_len = match file.metadata() {
Ok(m) => m.len(),
Err(e) => {
send_cancelable(&tx, IndexerMessage::Error {
generation,
message: e.to_string(),
}, &cancel_rx);
send_cancelable(
&tx,
IndexerMessage::Error {
generation,
message: e.to_string(),
},
&cancel_rx,
);
return;
}
};
@@ -321,10 +311,14 @@ pub fn spawn_indexer(
let buf = match buf_reader.fill_buf() {
Ok(b) => b,
Err(e) => {
send_cancelable(&tx, IndexerMessage::Error {
generation,
message: e.to_string(),
}, &cancel_rx);
send_cancelable(
&tx,
IndexerMessage::Error {
generation,
message: e.to_string(),
},
&cancel_rx,
);
return;
}
};
@@ -407,18 +401,26 @@ pub fn spawn_indexer(
Ok(_) | Err(_) => None,
},
Err(e) => {
send_cancelable(&tx, IndexerMessage::Error {
generation,
message: e.to_string(),
}, &cancel_rx);
send_cancelable(
&tx,
IndexerMessage::Error {
generation,
message: e.to_string(),
},
&cancel_rx,
);
return;
}
},
Err(e) => {
send_cancelable(&tx, IndexerMessage::Error {
generation,
message: e.to_string(),
}, &cancel_rx);
send_cancelable(
&tx,
IndexerMessage::Error {
generation,
message: e.to_string(),
},
&cancel_rx,
);
return;
}
}
@@ -430,18 +432,15 @@ pub fn spawn_indexer(
let reader = FileReader::from_parts(path, mmap, line_index);
let visual_height_index = if terminal_width > 0 {
let visual_heights = compute_visual_heights(&reader, terminal_width, json_format);
Some(VisualHeightIndex::build(&visual_heights).with_params(json_format, terminal_width))
} else {
None
};
send_cancelable(&tx, IndexerMessage::Complete {
generation,
reader,
visual_height_index,
}, &cancel_rx);
send_cancelable(
&tx,
IndexerMessage::Complete {
generation,
reader,
visual_height_index: None,
},
&cancel_rx,
);
});
rx
@@ -467,41 +466,67 @@ pub fn spawn_visual_height_rebuild(
Err(_) => return,
};
let mut reader = std::io::BufReader::with_capacity(64 * 1024, file);
let mut visual_heights = Vec::with_capacity(line_index.line_count());
let mut line_buf = Vec::new();
let mmap = match unsafe { memmap2::Mmap::map(&file) } {
Ok(m) => match file.metadata() {
Ok(meta) if meta.len() >= m.len() as u64 => m,
_ => return,
},
Err(_) => return,
};
let data = mmap.as_ref();
let total_lines = line_index.line_count();
let mut visual_heights: Vec<usize> = Vec::with_capacity(total_lines);
let mut line_start = 0usize;
let mut bytes_since_cancel_check = 0usize;
loop {
if cancel_rx.try_recv().is_ok() {
return;
if bytes_since_cancel_check >= 1_000_000 {
bytes_since_cancel_check = 0;
if cancel_rx.try_recv().is_ok() {
return;
}
}
line_buf.clear();
match std::io::BufRead::read_until(&mut reader, b'\n', &mut line_buf) {
Ok(0) => break,
Ok(_) => {
let line_text = std::str::from_utf8(&line_buf)
.ok()
.map(|s| s.trim_end_matches(['\r', '\n']))
.unwrap_or("");
visual_heights.push(compute_line_visual_height(
line_text,
terminal_width,
json_format,
));
}
Err(_) => return,
let newline_rel = memchr::memchr(b'\n', &data[line_start..]);
let line_end = match newline_rel {
Some(p) => line_start + p,
None => data.len(),
};
let mut slice_end = line_end;
if slice_end > line_start && data[slice_end - 1] == b'\r' {
slice_end -= 1;
}
let line_text = std::str::from_utf8(&data[line_start..slice_end]).unwrap_or("");
visual_heights.push(compute_line_visual_height(
line_text,
terminal_width,
json_format,
));
bytes_since_cancel_check =
bytes_since_cancel_check.saturating_add(line_end - line_start + 1);
match newline_rel {
Some(_) => line_start = line_end + 1,
None => break,
}
}
if visual_heights.len() != line_index.line_count() {
if visual_heights.len() != total_lines {
return;
}
let index =
VisualHeightIndex::build(&visual_heights).with_params(json_format, terminal_width);
send_cancelable(&tx, VisualHeightRebuildResult { generation, index }, &cancel_rx);
send_cancelable(
&tx,
VisualHeightRebuildResult { generation, index },
&cancel_rx,
);
});
rx
@@ -1114,12 +1139,10 @@ mod tests {
assert_eq!(reader.get_line(0), Some("line1"));
assert_eq!(reader.get_line(1), Some("line2"));
assert_eq!(reader.get_line(2), Some("line3"));
assert!(visual_height_index.is_some());
let idx = visual_height_index.unwrap();
assert_eq!(idx.total_visual_rows(), 3);
assert_eq!(idx.visual_row_to_logical_row(0), 0);
assert_eq!(idx.visual_row_to_logical_row(1), 1);
assert_eq!(idx.visual_row_to_logical_row(2), 2);
assert!(
visual_height_index.is_none(),
"spawn_indexer no longer builds VHI inline; UI triggers rebuild post-layout"
);
}
other => panic!("expected Complete, got {:?}", other),
}
@@ -1320,12 +1343,15 @@ mod tests {
match rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap() {
IndexerMessage::Progress { .. } => continue,
IndexerMessage::Complete {
reader,
visual_height_index,
..
} => {
let idx = visual_height_index.expect("should have visual height index");
assert_eq!(idx.visual_height_of_line(0), 1);
assert_eq!(idx.visual_height_of_line(1), 1);
assert_eq!(reader.line_count(), 2);
assert!(
visual_height_index.is_none(),
"spawn_indexer no longer builds VHI inline; UI triggers rebuild post-layout"
);
break;
}
other => panic!("expected Complete, got {:?}", other),
@@ -1515,13 +1541,7 @@ mod tests {
}
let (_cancel_tx, cancel_rx) = crossbeam_channel::bounded(1);
let rx = spawn_visual_height_rebuild(
f.path().to_path_buf(),
1,
80,
false,
cancel_rx,
);
let rx = spawn_visual_height_rebuild(f.path().to_path_buf(), 1, 80, false, cancel_rx);
let result = rx.recv_timeout(std::time::Duration::from_secs(5));
match result {
@@ -1604,7 +1624,10 @@ mod tests {
Err(e) => panic!("recv error: {:?}", e),
}
}
assert!(got_complete, "indexer should complete even when Progress fills channel");
assert!(
got_complete,
"indexer should complete even when Progress fills channel"
);
}
#[test]
crates/core/src/io/wrap.rs
+194 -12
View File
@@ -1,3 +1,5 @@
use std::borrow::Cow;
/// Maximum input length for wrap/format operations (10 MB).
/// Callers should check against this constant before invoking `wrap_line_chars`
/// to avoid pathological cases on oversized lines.
@@ -69,21 +71,81 @@ pub fn wrap_line_chars(line: &str, width: usize) -> Vec<String> {
result
}
/// Format a line as pretty-printed JSON if it's a JSON Object.
/// Returns the original line unchanged for non-JSON or non-Object content.
pub fn format_json_line(line: &str) -> String {
if line.trim().is_empty() {
return String::new();
/// Count wrapped rows for a line without allocating the wrapped strings.
/// MUST produce the same count as `wrap_line_chars(line, width).len()`.
pub fn wrap_line_count(line: &str, width: usize) -> usize {
use unicode_width::UnicodeWidthChar;
if width == 0 || line.is_empty() {
return 1;
}
// Quick pre-check: only try parsing if it starts with '{'
if !line.trim_start().starts_with('{') {
return line.to_string();
let mut count = 0usize;
let mut col = 0usize;
let mut row_has_content = false;
for ch in line.chars() {
if ch == '\t' {
let tab_stop = TAB_WIDTH - (col % TAB_WIDTH);
let mut remaining = tab_stop;
while remaining > 0 {
let avail = width.saturating_sub(col);
let fill = remaining.min(avail);
col += fill;
remaining -= fill;
if col >= width {
count += 1;
col = 0;
row_has_content = false;
} else {
row_has_content = true;
}
}
} else {
let w = if ch.is_control() {
0
} else {
ch.width().unwrap_or(0)
};
if col + w > width && row_has_content {
count += 1;
col = 0;
}
col += w;
if col >= width {
count += 1;
col = 0;
row_has_content = false;
} else {
row_has_content = true;
}
}
}
if row_has_content {
count += 1;
}
if count == 0 {
count = 1;
}
count
}
/// Format a line as pretty-printed JSON if it's a JSON Object.
/// Returns the original line borrowed for non-JSON or non-Object content,
/// only allocating when pretty-printing actually happens.
pub fn format_json_line(line: &str) -> Cow<'_, str> {
let trimmed = line.trim();
if trimmed.is_empty() {
return Cow::Borrowed("");
}
if !trimmed.starts_with('{') {
return Cow::Borrowed(line);
}
match serde_json::from_str::<serde_json::Value>(line) {
Ok(value) if value.is_object() => {
serde_json::to_string_pretty(&value).unwrap_or_else(|_| line.to_string())
}
_ => line.to_string(),
Ok(value) if value.is_object() => match serde_json::to_string_pretty(&value) {
Ok(s) => Cow::Owned(s),
Err(_) => Cow::Borrowed(line),
},
_ => Cow::Borrowed(line),
}
}
@@ -231,4 +293,124 @@ mod tests {
let result = wrap_line_chars("ab\t", 4);
assert_eq!(result, vec!["ab "]);
}
#[test]
fn test_wrap_count_matches_chars_empty() {
assert_eq!(wrap_line_count("", 80), wrap_line_chars("", 80).len());
}
#[test]
fn test_wrap_count_matches_chars_zero_width() {
assert_eq!(
wrap_line_count("hello", 0),
wrap_line_chars("hello", 0).len()
);
}
#[test]
fn test_wrap_count_matches_chars_ascii_short() {
for width in 1..20 {
assert_eq!(
wrap_line_count("hello world", width),
wrap_line_chars("hello world", width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_cjk() {
for width in 1..10 {
assert_eq!(
wrap_line_count("你好世界", width),
wrap_line_chars("你好世界", width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_emoji() {
for width in 1..6 {
assert_eq!(
wrap_line_count("😀👨‍👩‍👧", width),
wrap_line_chars("😀👨‍👩‍👧", width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_combining_mark() {
for width in 1..6 {
assert_eq!(
wrap_line_count("a\u{0301}b\u{0301}c", width),
wrap_line_chars("a\u{0301}b\u{0301}c", width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_tab_variants() {
let inputs = ["\t", "a\tb", "ab\t", "\t\t", "abc\tb", "a\t\t\tb"];
for input in inputs {
for width in 1..9 {
assert_eq!(
wrap_line_count(input, width),
wrap_line_chars(input, width).len(),
"input={input:?} width={width}"
);
}
}
}
#[test]
fn test_wrap_count_matches_chars_mixed() {
let input = "log: 你好 😀\tvalue=true [ERROR]";
for width in 1..40 {
assert_eq!(
wrap_line_count(input, width),
wrap_line_chars(input, width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_zero_width_chars() {
let input = "a\u{200B}b\u{200C}c\u{200D}d";
for width in 1..8 {
assert_eq!(
wrap_line_count(input, width),
wrap_line_chars(input, width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_control_chars() {
let input = "a\x07b\x1Fc";
for width in 1..8 {
assert_eq!(
wrap_line_count(input, width),
wrap_line_chars(input, width).len(),
"width={width}"
);
}
}
#[test]
fn test_wrap_count_matches_chars_long_log_line() {
let input = "2025-01-14T10:23:45.123Z [INFO] [auth] request handled status=200 \
latency=0.234s endpoint=/api/v1/users user=user_42 request_id=req_abc123def456";
for width in [10, 20, 40, 60, 80, 100, 120] {
assert_eq!(
wrap_line_count(input, width),
wrap_line_chars(input, width).len(),
"width={width}"
);
}
}
}