diff --git a/crates/core/src/io/progressive_reader.rs b/crates/core/src/io/progressive_reader.rs index 4a1f70a..c434090 100644 --- a/crates/core/src/io/progressive_reader.rs +++ b/crates/core/src/io/progressive_reader.rs @@ -8,7 +8,7 @@ use crate::io::file_reader::{AppendStatus, FileReader}; use crate::io::index_cache::IndexCache; use crate::io::line_index::LineIndex; use crate::io::line_sampler::sample_line_count; -use crate::io::wrap::{format_json_line, wrap_line_chars, MAX_WRAP_INPUT_LEN}; +use crate::io::wrap::{MAX_WRAP_INPUT_LEN, format_json_line}; // ─── Cancel-aware channel helpers ──────────────────────────────────────────── @@ -235,29 +235,11 @@ pub fn compute_line_visual_height( fn compute_text_visual_height(text: &str, width: usize) -> usize { let mut height = 0; for sub_line in text.split('\n') { - height += wrap_line_chars(sub_line, width).len(); + height += crate::io::wrap::wrap_line_count(sub_line, width); } height.max(1) } -fn compute_visual_heights( - reader: &FileReader, - terminal_width: usize, - json_format: bool, -) -> Vec { - let line_count = reader.line_count(); - let mut visual_heights = Vec::with_capacity(line_count); - for i in 0..line_count { - let line_text = reader.get_line(i).unwrap_or(""); - visual_heights.push(compute_line_visual_height( - line_text, - terminal_width, - json_format, - )); - } - visual_heights -} - // ─── ReaderState ───────────────────────────────────────────────────────────── pub enum ReaderState { @@ -281,8 +263,8 @@ pub enum ReaderState { pub fn spawn_indexer( path: PathBuf, generation: u64, - terminal_width: usize, - json_format: bool, + _terminal_width: usize, + _json_format: bool, cancel_rx: crossbeam_channel::Receiver<()>, ) -> crossbeam_channel::Receiver { let (tx, rx) = crossbeam_channel::bounded(10); @@ -291,20 +273,28 @@ pub fn spawn_indexer( let file = match std::fs::File::open(&path) { Ok(f) => f, Err(e) => { - send_cancelable(&tx, IndexerMessage::Error { - generation, - message: e.to_string(), - }, &cancel_rx); + send_cancelable( + &tx, + IndexerMessage::Error { + generation, + message: e.to_string(), + }, + &cancel_rx, + ); return; } }; let target_len = match file.metadata() { Ok(m) => m.len(), Err(e) => { - send_cancelable(&tx, IndexerMessage::Error { - generation, - message: e.to_string(), - }, &cancel_rx); + send_cancelable( + &tx, + IndexerMessage::Error { + generation, + message: e.to_string(), + }, + &cancel_rx, + ); return; } }; @@ -321,10 +311,14 @@ pub fn spawn_indexer( let buf = match buf_reader.fill_buf() { Ok(b) => b, Err(e) => { - send_cancelable(&tx, IndexerMessage::Error { - generation, - message: e.to_string(), - }, &cancel_rx); + send_cancelable( + &tx, + IndexerMessage::Error { + generation, + message: e.to_string(), + }, + &cancel_rx, + ); return; } }; @@ -407,18 +401,26 @@ pub fn spawn_indexer( Ok(_) | Err(_) => None, }, Err(e) => { - send_cancelable(&tx, IndexerMessage::Error { - generation, - message: e.to_string(), - }, &cancel_rx); + send_cancelable( + &tx, + IndexerMessage::Error { + generation, + message: e.to_string(), + }, + &cancel_rx, + ); return; } }, Err(e) => { - send_cancelable(&tx, IndexerMessage::Error { - generation, - message: e.to_string(), - }, &cancel_rx); + send_cancelable( + &tx, + IndexerMessage::Error { + generation, + message: e.to_string(), + }, + &cancel_rx, + ); return; } } @@ -430,18 +432,15 @@ pub fn spawn_indexer( let reader = FileReader::from_parts(path, mmap, line_index); - let visual_height_index = if terminal_width > 0 { - let visual_heights = compute_visual_heights(&reader, terminal_width, json_format); - Some(VisualHeightIndex::build(&visual_heights).with_params(json_format, terminal_width)) - } else { - None - }; - - send_cancelable(&tx, IndexerMessage::Complete { - generation, - reader, - visual_height_index, - }, &cancel_rx); + send_cancelable( + &tx, + IndexerMessage::Complete { + generation, + reader, + visual_height_index: None, + }, + &cancel_rx, + ); }); rx @@ -467,41 +466,67 @@ pub fn spawn_visual_height_rebuild( Err(_) => return, }; - let mut reader = std::io::BufReader::with_capacity(64 * 1024, file); - let mut visual_heights = Vec::with_capacity(line_index.line_count()); - let mut line_buf = Vec::new(); + let mmap = match unsafe { memmap2::Mmap::map(&file) } { + Ok(m) => match file.metadata() { + Ok(meta) if meta.len() >= m.len() as u64 => m, + _ => return, + }, + Err(_) => return, + }; + let data = mmap.as_ref(); + let total_lines = line_index.line_count(); + let mut visual_heights: Vec = Vec::with_capacity(total_lines); + + let mut line_start = 0usize; + let mut bytes_since_cancel_check = 0usize; loop { - if cancel_rx.try_recv().is_ok() { - return; + if bytes_since_cancel_check >= 1_000_000 { + bytes_since_cancel_check = 0; + if cancel_rx.try_recv().is_ok() { + return; + } } - line_buf.clear(); - match std::io::BufRead::read_until(&mut reader, b'\n', &mut line_buf) { - Ok(0) => break, - Ok(_) => { - let line_text = std::str::from_utf8(&line_buf) - .ok() - .map(|s| s.trim_end_matches(['\r', '\n'])) - .unwrap_or(""); - visual_heights.push(compute_line_visual_height( - line_text, - terminal_width, - json_format, - )); - } - Err(_) => return, + let newline_rel = memchr::memchr(b'\n', &data[line_start..]); + let line_end = match newline_rel { + Some(p) => line_start + p, + None => data.len(), + }; + + let mut slice_end = line_end; + if slice_end > line_start && data[slice_end - 1] == b'\r' { + slice_end -= 1; + } + + let line_text = std::str::from_utf8(&data[line_start..slice_end]).unwrap_or(""); + visual_heights.push(compute_line_visual_height( + line_text, + terminal_width, + json_format, + )); + + bytes_since_cancel_check = + bytes_since_cancel_check.saturating_add(line_end - line_start + 1); + + match newline_rel { + Some(_) => line_start = line_end + 1, + None => break, } } - if visual_heights.len() != line_index.line_count() { + if visual_heights.len() != total_lines { return; } let index = VisualHeightIndex::build(&visual_heights).with_params(json_format, terminal_width); - send_cancelable(&tx, VisualHeightRebuildResult { generation, index }, &cancel_rx); + send_cancelable( + &tx, + VisualHeightRebuildResult { generation, index }, + &cancel_rx, + ); }); rx @@ -1114,12 +1139,10 @@ mod tests { assert_eq!(reader.get_line(0), Some("line1")); assert_eq!(reader.get_line(1), Some("line2")); assert_eq!(reader.get_line(2), Some("line3")); - assert!(visual_height_index.is_some()); - let idx = visual_height_index.unwrap(); - assert_eq!(idx.total_visual_rows(), 3); - assert_eq!(idx.visual_row_to_logical_row(0), 0); - assert_eq!(idx.visual_row_to_logical_row(1), 1); - assert_eq!(idx.visual_row_to_logical_row(2), 2); + assert!( + visual_height_index.is_none(), + "spawn_indexer no longer builds VHI inline; UI triggers rebuild post-layout" + ); } other => panic!("expected Complete, got {:?}", other), } @@ -1320,12 +1343,15 @@ mod tests { match rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap() { IndexerMessage::Progress { .. } => continue, IndexerMessage::Complete { + reader, visual_height_index, .. } => { - let idx = visual_height_index.expect("should have visual height index"); - assert_eq!(idx.visual_height_of_line(0), 1); - assert_eq!(idx.visual_height_of_line(1), 1); + assert_eq!(reader.line_count(), 2); + assert!( + visual_height_index.is_none(), + "spawn_indexer no longer builds VHI inline; UI triggers rebuild post-layout" + ); break; } other => panic!("expected Complete, got {:?}", other), @@ -1515,13 +1541,7 @@ mod tests { } let (_cancel_tx, cancel_rx) = crossbeam_channel::bounded(1); - let rx = spawn_visual_height_rebuild( - f.path().to_path_buf(), - 1, - 80, - false, - cancel_rx, - ); + let rx = spawn_visual_height_rebuild(f.path().to_path_buf(), 1, 80, false, cancel_rx); let result = rx.recv_timeout(std::time::Duration::from_secs(5)); match result { @@ -1604,7 +1624,10 @@ mod tests { Err(e) => panic!("recv error: {:?}", e), } } - assert!(got_complete, "indexer should complete even when Progress fills channel"); + assert!( + got_complete, + "indexer should complete even when Progress fills channel" + ); } #[test] diff --git a/crates/core/src/io/wrap.rs b/crates/core/src/io/wrap.rs index e4ad23a..3f6557d 100644 --- a/crates/core/src/io/wrap.rs +++ b/crates/core/src/io/wrap.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + /// Maximum input length for wrap/format operations (10 MB). /// Callers should check against this constant before invoking `wrap_line_chars` /// to avoid pathological cases on oversized lines. @@ -69,21 +71,81 @@ pub fn wrap_line_chars(line: &str, width: usize) -> Vec { result } -/// Format a line as pretty-printed JSON if it's a JSON Object. -/// Returns the original line unchanged for non-JSON or non-Object content. -pub fn format_json_line(line: &str) -> String { - if line.trim().is_empty() { - return String::new(); +/// Count wrapped rows for a line without allocating the wrapped strings. +/// MUST produce the same count as `wrap_line_chars(line, width).len()`. +pub fn wrap_line_count(line: &str, width: usize) -> usize { + use unicode_width::UnicodeWidthChar; + + if width == 0 || line.is_empty() { + return 1; } - // Quick pre-check: only try parsing if it starts with '{' - if !line.trim_start().starts_with('{') { - return line.to_string(); + let mut count = 0usize; + let mut col = 0usize; + let mut row_has_content = false; + + for ch in line.chars() { + if ch == '\t' { + let tab_stop = TAB_WIDTH - (col % TAB_WIDTH); + let mut remaining = tab_stop; + while remaining > 0 { + let avail = width.saturating_sub(col); + let fill = remaining.min(avail); + col += fill; + remaining -= fill; + if col >= width { + count += 1; + col = 0; + row_has_content = false; + } else { + row_has_content = true; + } + } + } else { + let w = if ch.is_control() { + 0 + } else { + ch.width().unwrap_or(0) + }; + if col + w > width && row_has_content { + count += 1; + col = 0; + } + col += w; + if col >= width { + count += 1; + col = 0; + row_has_content = false; + } else { + row_has_content = true; + } + } + } + if row_has_content { + count += 1; + } + if count == 0 { + count = 1; + } + count +} + +/// Format a line as pretty-printed JSON if it's a JSON Object. +/// Returns the original line borrowed for non-JSON or non-Object content, +/// only allocating when pretty-printing actually happens. +pub fn format_json_line(line: &str) -> Cow<'_, str> { + let trimmed = line.trim(); + if trimmed.is_empty() { + return Cow::Borrowed(""); + } + if !trimmed.starts_with('{') { + return Cow::Borrowed(line); } match serde_json::from_str::(line) { - Ok(value) if value.is_object() => { - serde_json::to_string_pretty(&value).unwrap_or_else(|_| line.to_string()) - } - _ => line.to_string(), + Ok(value) if value.is_object() => match serde_json::to_string_pretty(&value) { + Ok(s) => Cow::Owned(s), + Err(_) => Cow::Borrowed(line), + }, + _ => Cow::Borrowed(line), } } @@ -231,4 +293,124 @@ mod tests { let result = wrap_line_chars("ab\t", 4); assert_eq!(result, vec!["ab "]); } + + #[test] + fn test_wrap_count_matches_chars_empty() { + assert_eq!(wrap_line_count("", 80), wrap_line_chars("", 80).len()); + } + + #[test] + fn test_wrap_count_matches_chars_zero_width() { + assert_eq!( + wrap_line_count("hello", 0), + wrap_line_chars("hello", 0).len() + ); + } + + #[test] + fn test_wrap_count_matches_chars_ascii_short() { + for width in 1..20 { + assert_eq!( + wrap_line_count("hello world", width), + wrap_line_chars("hello world", width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_cjk() { + for width in 1..10 { + assert_eq!( + wrap_line_count("你好世界", width), + wrap_line_chars("你好世界", width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_emoji() { + for width in 1..6 { + assert_eq!( + wrap_line_count("😀👨‍👩‍👧", width), + wrap_line_chars("😀👨‍👩‍👧", width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_combining_mark() { + for width in 1..6 { + assert_eq!( + wrap_line_count("a\u{0301}b\u{0301}c", width), + wrap_line_chars("a\u{0301}b\u{0301}c", width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_tab_variants() { + let inputs = ["\t", "a\tb", "ab\t", "\t\t", "abc\tb", "a\t\t\tb"]; + for input in inputs { + for width in 1..9 { + assert_eq!( + wrap_line_count(input, width), + wrap_line_chars(input, width).len(), + "input={input:?} width={width}" + ); + } + } + } + + #[test] + fn test_wrap_count_matches_chars_mixed() { + let input = "log: 你好 😀\tvalue=true [ERROR]"; + for width in 1..40 { + assert_eq!( + wrap_line_count(input, width), + wrap_line_chars(input, width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_zero_width_chars() { + let input = "a\u{200B}b\u{200C}c\u{200D}d"; + for width in 1..8 { + assert_eq!( + wrap_line_count(input, width), + wrap_line_chars(input, width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_control_chars() { + let input = "a\x07b\x1Fc"; + for width in 1..8 { + assert_eq!( + wrap_line_count(input, width), + wrap_line_chars(input, width).len(), + "width={width}" + ); + } + } + + #[test] + fn test_wrap_count_matches_chars_long_log_line() { + let input = "2025-01-14T10:23:45.123Z [INFO] [auth] request handled status=200 \ + latency=0.234s endpoint=/api/v1/users user=user_42 request_id=req_abc123def456"; + for width in [10, 20, 40, 60, 80, 100, 120] { + assert_eq!( + wrap_line_count(input, width), + wrap_line_chars(input, width).len(), + "width={width}" + ); + } + } }