feat(tui): replace O(N) scanning with progressive loading and VisualHeightIndex

Replace synchronous file loading with AppLoadingState state machine (Empty/Loading/Ready/Error) for instant interactivity. Add ViewportCache for on-demand viewport computation, replacing global wrap/level caches. Integrate background indexer polling and file watcher events into the TUI event loop. Add loading UI with progress percentage, estimated line numbers with ~ prefix, and error state display. Eliminate all O(N) linear scans using VisualHeightIndex binary search.

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

crates/tui/Cargo.toml

@@ -10,6 +10,7 @@ clap.workspace = true
 anyhow.workspace = true
 log-viewer-core.workspace = true
 serde_json.workspace = true
+crossbeam-channel.workspace = true
 
 [dev-dependencies]
 tempfile = { workspace = true }

crates/tui/src/main.rs

@@ -31,6 +31,8 @@ fn main() -> anyhow::Result<()> {
     }
 
     while !app.should_quit {
+        app.poll_background_indexer();
+        app.poll_file_watcher();
         terminal.draw(|frame| ui::render(frame, &mut app))?;
         if crossterm::event::poll(std::time::Duration::from_millis(100))? {
             match crossterm::event::read()? {

crates/tui/src/ui.rs

@@ -1,4 +1,4 @@
-use ratatui::style::{Color, Style};
+use ratatui::style::{Color, Modifier, Style};
 use ratatui::text::{Line, Span};
 
 use crate::app::{App, AppMode};
@@ -42,6 +42,10 @@ pub fn render(frame: &mut ratatui::Frame, app: &mut App) {
     // ── Title bar ──────────────────────────────────────────────────
     let title_text = if app.mode == AppMode::Settings {
         " Color Settings".to_string()
+    } else if app.is_loading() {
+        let name = app.file_name().unwrap_or("unknown");
+        let pct = app.loading_progress().map_or(0, |p| p as usize);
+        format!(" {} [Loading... {}%]", name, pct)
     } else if app.is_loaded() {
         let name = app.file_name().unwrap_or("unknown");
         let cursor_display = if app.total_lines() == 0 {
@@ -61,6 +65,22 @@ pub fn render(frame: &mut ratatui::Frame, app: &mut App) {
     // ── Content area ───────────────────────────────────────────────
     if app.mode == AppMode::Settings {
         render_settings(frame, app, outer[1]);
+    } else if app.is_error() {
+        let msg = app.error_message().unwrap_or_default();
+        let error_lines = vec![
+            Line::from(""),
+            Line::from(""),
+            Line::styled(
+                format!(" Error: {}", msg),
+                Style::default().fg(Color::Red).add_modifier(Modifier::BOLD),
+            ),
+            Line::from(""),
+            Line::styled(" Press q to quit", Style::default().fg(Color::DarkGray)),
+        ];
+        frame.render_widget(Paragraph::new(error_lines).centered(), outer[1]);
+    } else if app.is_loading() {
+        // Show content from sampling during loading
+        render_content(frame, app, outer[1]);
     } else if !app.is_loaded() {
         frame.render_widget(Paragraph::new(" No file loaded"), outer[1]);
     } else {
@@ -70,6 +90,30 @@ pub fn render(frame: &mut ratatui::Frame, app: &mut App) {
     // ── Status bar ─────────────────────────────────────────────────
     let status_text = if app.mode == AppMode::Settings {
         " j/k:navigate  ←/→:change  1-8:jump  Enter:save  Esc:cancel"
+    } else if app.is_error() {
+        " Press q to quit"
+    } else if app.is_loading() {
+        let pct = app.loading_progress().map_or(0, |p| p as usize);
+        let est = app
+            .estimated_lines()
+            .map_or("?".to_string(), |e| format!("~{}", e));
+        let name = app.file_name().unwrap_or("unknown");
+        let status = format!(
+            " Indexing... {}% | {} lines | {} | j/k:scroll q:quit",
+            pct, est, name
+        );
+        frame.render_widget(
+            Paragraph::new(status).style(Style::default().fg(Color::Yellow)),
+            outer[2],
+        );
+        return;
+    } else if app.is_loaded() {
+        let name = app.file_name().unwrap_or("unknown");
+        let total = app.total_lines();
+        let cursor_display = if total == 0 { 0 } else { app.cursor_line + 1 };
+        let status = format!(" {} [{}/{}] | j/k:scroll d/u:half-page f/b:page G/gg:jump Tab:format S:settings q:quit", name, cursor_display, total);
+        frame.render_widget(Paragraph::new(status), outer[2]);
+        return;
     } else {
         " j/k:scroll d/u:half-page f/b:page G/gg:jump Tab:format S:settings q:quit"
     };
@@ -165,8 +209,11 @@ fn render_content(frame: &mut ratatui::Frame, app: &mut App, area: ratatui::layo
     } else {
         0
     };
+
+    let is_loading = app.is_loading();
+    let gutter_prefix_extra = if is_loading { 1 } else { 0 };
     let gutter_width = if total_lines > 0 {
-        line_num_width + 1 + 1
+        line_num_width + gutter_prefix_extra + 1 + 1
     } else {
         0
     };
@@ -177,62 +224,73 @@ fn render_content(frame: &mut ratatui::Frame, app: &mut App, area: ratatui::layo
         return;
     }
 
-    app.recompute_wrap_cache(actual_content_width);
+    let (start_logical, offset_in_line) = app.ensure_viewport_cache(actual_content_width);
-    let mut visual_acc: usize = 0;
-    let mut start_logical: usize = 0;
-    let mut offset_in_line: usize = 0;
-    let v_offset = app.v_offset;
-
-    for (i, &h) in app.visual_heights.iter().enumerate() {
-        if visual_acc.saturating_add(h) > v_offset {
-            start_logical = i;
-            offset_in_line = v_offset.saturating_sub(visual_acc);
-            break;
-        }
-        visual_acc += h;
-        if i == app.visual_heights.len() - 1 {
-            start_logical = i;
-            offset_in_line = 0;
-        }
-    }
 
     let mut lines: Vec<Line> = Vec::new();
     let mut current_visual_offset: usize = 0;
     let available_rows = content_height;
 
-    for logical_line in start_logical..total_lines {
+    let gutter_style = if is_loading {
-        let wrapped = &app.wrap_cache[logical_line];
+        Style::default()
+            .fg(Color::DarkGray)
+            .add_modifier(Modifier::DIM)
+    } else {
+        Style::default().fg(Color::DarkGray)
+    };
+
+    for (entry_idx, entry) in app.viewport_cache.entries.iter().enumerate() {
+        let logical_line = app.viewport_cache.logical_start + entry_idx;
         let start_row = if logical_line == start_logical {
             offset_in_line
         } else {
             0
         };
 
-        for (visual_row, text) in wrapped.iter().enumerate().skip(start_row) {
+        for (visual_row, text) in entry.wrapped_rows.iter().enumerate().skip(start_row) {
             if current_visual_offset >= available_rows {
                 break;
             }
 
             let is_cursor = logical_line == app.cursor_line;
-            let level = app.level_cache.get(logical_line).and_then(|l| l.as_ref());
+            let level = entry.level.as_ref();
+
+            let bg_color = if is_cursor {
+                Color::DarkGray
+            } else {
+                Color::Reset
+            };
+            let level_fg = level_fg(level, &app.color_config).unwrap_or(Color::White);
 
             let gutter_text = if visual_row == 0 {
+                if is_loading {
+                    format!(
+                        "~{:>width$} \u{2502}",
+                        logical_line + 1,
+                        width = line_num_width
+                    )
+                } else {
                     format!(
                         "{:>width$} \u{2502}",
                         logical_line + 1,
                         width = line_num_width
                     )
+                }
+            } else if is_loading {
+                format!(" {:width$} \u{2502}", "", width = line_num_width)
             } else {
                 format!("{:width$} \u{2502}", "", width = line_num_width)
             };
 
-            lines.push(build_line_spans(
+            let effective_gutter_style = if is_cursor {
-                gutter_text,
+                gutter_style.bg(bg_color)
-                text.clone(),
+            } else {
-                is_cursor,
+                gutter_style
-                level,
+            };
-                &app.color_config,
+
-            ));
+            lines.push(Line::from(vec![
+                Span::styled(gutter_text, effective_gutter_style),
+                Span::styled(text.clone(), Style::default().fg(level_fg).bg(bg_color)),
+            ]));
 
             current_visual_offset += 1;
         }
@@ -304,4 +362,135 @@ mod tests {
         assert_eq!(line.spans[0].style.bg, Some(Color::Reset));
         assert_eq!(line.spans[1].style.bg, Some(Color::Reset));
     }
+
+    fn render_to_buffer(app: &mut App, width: u16, height: u16) -> ratatui::buffer::Buffer {
+        let backend = ratatui::backend::TestBackend::new(width, height);
+        let mut terminal = ratatui::Terminal::new(backend).unwrap();
+        terminal.draw(|frame| render(frame, app)).unwrap();
+        terminal.backend().buffer().clone()
+    }
+
+    fn make_temp_file(content: &str) -> std::path::PathBuf {
+        use std::sync::atomic::{AtomicU64, Ordering};
+        static COUNTER: AtomicU64 = AtomicU64::new(0);
+        let id = COUNTER.fetch_add(1, Ordering::Relaxed);
+        let dir = std::env::temp_dir();
+        let name = format!("log_viewer_ui_test_{}_{}", std::process::id(), id);
+        let path = dir.join(name);
+        std::fs::write(&path, content).unwrap();
+        path
+    }
+
+    #[test]
+    fn test_render_error_state() {
+        let mut app = App::new();
+        app.set_error_state("file not found");
+
+        let buf = render_to_buffer(&mut app, 80, 24);
+
+        let mut found_error = false;
+        let mut found_quit = false;
+        for row in 0..23 {
+            let content: String = (0..80)
+                .map(|c| buf.cell((c, row)).unwrap().symbol().to_string())
+                .collect();
+            if content.contains("Error") && content.contains("file not found") {
+                found_error = true;
+            }
+            if content.contains("Press q to quit") {
+                found_quit = true;
+            }
+        }
+        assert!(found_error, "content area should show error message");
+        assert!(found_quit, "content area should show quit hint");
+    }
+
+    #[test]
+    fn test_render_loading_state_status_bar() {
+        let path = make_temp_file("line1\nline2\nline3\nline4\nline5\n");
+        let result = std::panic::catch_unwind(|| {
+            let mut app = App::new();
+            app.load_file(path.to_str().unwrap()).unwrap();
+
+            if app.is_loading() {
+                let buf = render_to_buffer(&mut app, 80, 24);
+
+                let status: String = (0..80)
+                    .map(|c| buf.cell((c, 23)).unwrap().symbol().to_string())
+                    .collect();
+                assert!(
+                    status.contains("Indexing"),
+                    "status bar should show 'Indexing', got: {}",
+                    status
+                );
+                assert!(
+                    status.contains("%"),
+                    "status bar should show percentage, got: {}",
+                    status
+                );
+                assert!(
+                    status.contains("~"),
+                    "status bar should show ~ for estimated lines, got: {}",
+                    status
+                );
+            }
+        });
+        let _ = std::fs::remove_file(&path);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_render_loading_gutter_tilde_prefix() {
+        let path = make_temp_file("line1\nline2\nline3\n");
+        let result = std::panic::catch_unwind(|| {
+            let mut app = App::new();
+            app.load_file(path.to_str().unwrap()).unwrap();
+
+            if app.is_loading() {
+                let buf = render_to_buffer(&mut app, 80, 24);
+
+                let first_line: String = (0..10)
+                    .map(|c| buf.cell((c, 1)).unwrap().symbol().to_string())
+                    .collect();
+                assert!(
+                    first_line.contains("~"),
+                    "loading state gutter should have ~ prefix, got: {}",
+                    first_line
+                );
+            }
+        });
+        let _ = std::fs::remove_file(&path);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_render_ready_state_status_bar() {
+        let path = make_temp_file("alpha\nbeta\ngamma\n");
+        let result = std::panic::catch_unwind(|| {
+            let data = std::fs::read(&path).unwrap();
+            let index = log_viewer_core::io::line_index::LineIndex::from_bytes(&data);
+            let _ = log_viewer_core::io::index_cache::IndexCache::save(&path, &index);
+
+            let mut app = App::new();
+            app.load_file(path.to_str().unwrap()).unwrap();
+
+            assert!(
+                app.is_loaded() && !app.is_loading(),
+                "should be in Ready state with cache hit"
+            );
+
+            let buf = render_to_buffer(&mut app, 80, 24);
+
+            let status: String = (0..80)
+                .map(|c| buf.cell((c, 23)).unwrap().symbol().to_string())
+                .collect();
+            assert!(
+                status.contains("1/") || status.contains("1/3"),
+                "status bar should show cursor position, got: {}",
+                status
+            );
+        });
+        let _ = std::fs::remove_file(&path);
+        assert!(result.is_ok());
+    }
 }