//! Port of Durable's `LineScanner` (`env/line-scan.ts`): locates lines `[start_line, end_line)` of a file fed in order //! and measures their decoded UTF-8 size as `new TextDecoder().decode(file)` would produce it. use crate::decode::{decode_into, range_decoder, starts_with_bom}; use encoding_rs::Decoder; use serde_json::{Value, json}; const NEWLINE: u8 = 0x0a; pub struct LineScanner { start_line: u64, end_line: Option, position: u64, newlines: u64, line_start: u64, start: Option, end: Option, first_line_end: Option, last_line_start: Option, selected_bytes: u64, first_line_bytes: u64, selection: Option, first_line: Option, head: Option>, bom: bool, } impl LineScanner { /// `end_line`, when given, must be greater than `start_line`. pub fn new(start_line: u64, end_line: Option) -> LineScanner { let mut scanner = LineScanner { start_line, end_line, position: 0, newlines: 0, line_start: 0, start: None, end: None, first_line_end: None, last_line_start: None, selected_bytes: 0, first_line_bytes: 0, selection: None, first_line: None, head: Some(Vec::with_capacity(3)), bom: false, }; if start_line == 0 { scanner.begin(0); } scanner } fn is_last_line(&self, line: u64) -> bool { self.end_line.is_some_and(|end| line + 1 == end) } pub fn push(&mut self, mut chunk: &[u8]) { if let Some(head) = &mut self.head { let take = (3 - head.len()).min(chunk.len()); head.extend_from_slice(&chunk[..take]); if head.len() < 3 { return; } chunk = &chunk[take..]; self.release_head(); } self.process(chunk); } fn release_head(&mut self) { let head = self.head.take().unwrap_or_default(); self.bom = starts_with_bom(&head); self.process(&head); } fn process(&mut self, chunk: &[u8]) { let base = self.position; let mut from = 0; let mut index = 0; while let Some(found) = chunk[index..].iter().position(|byte| *byte == NEWLINE) { let newline = index + found; // The newline ends line `self.newlines`; it belongs to the selection between selected lines only. self.feed(chunk, base, from, newline); let line = self.newlines; let position = base + newline as u64; if line == self.start_line { self.end_first_line(position); } if self.is_last_line(line) { self.end_selection(position); } self.feed(chunk, base, newline, newline + 1); from = newline + 1; self.newlines += 1; self.line_start = position + 1; if self.newlines == self.start_line { self.begin(self.line_start); } if self.is_last_line(self.newlines) { self.last_line_start = Some(self.line_start); } index = newline + 1; } self.feed(chunk, base, from, chunk.len()); self.position += chunk.len() as u64; } pub fn finish(mut self) -> Value { if self.head.is_some() { self.release_head(); } let size = self.position; let Some(start) = self.start else { return json!({ "newlines": self.newlines, "start": size, "end": size, "firstLineEnd": size, "lastLineStart": size, "selectedBytes": 0, "firstLineBytes": 0, }); }; if self.first_line_end.is_none() { self.end_first_line(size); } if self.end.is_none() { self.end_selection(size); } json!({ "newlines": self.newlines, "start": start, "end": self.end.unwrap_or(size), "firstLineEnd": self.first_line_end.unwrap_or(size), // A selection that reaches past the last line ends with the last line. "lastLineStart": self.last_line_start.unwrap_or(self.line_start), "selectedBytes": self.selected_bytes, "firstLineBytes": self.first_line_bytes, }) } fn begin(&mut self, start: u64) { self.start = Some(start); if self.is_last_line(self.start_line) { self.last_line_start = Some(start); } self.selection = Some(range_decoder()); self.first_line = Some(range_decoder()); } fn end_first_line(&mut self, position: u64) { self.first_line_end = Some(position); if let Some(mut decoder) = self.first_line.take() { self.first_line_bytes += decode_into(&mut decoder, &[], true).len() as u64; } } fn end_selection(&mut self, position: u64) { self.end = Some(position); if let Some(mut decoder) = self.selection.take() { self.selected_bytes += decode_into(&mut decoder, &[], true).len() as u64; } } fn feed(&mut self, chunk: &[u8], base: u64, mut from: usize, to: usize) { // Decoding the whole file drops a leading byte-order mark. if self.bom && base + (from as u64) < 3 { from = to.min((3 - base) as usize); } if to <= from { return; } let bytes = &chunk[from..to]; if let Some(decoder) = &mut self.selection { self.selected_bytes += decode_into(decoder, bytes, false).len() as u64; } if let Some(decoder) = &mut self.first_line { self.first_line_bytes += decode_into(decoder, bytes, false).len() as u64; } } } #[cfg(test)] mod tests { use super::*; fn scan(file: &[u8], start: u64, end: Option, chunk: usize) -> Value { let mut scanner = LineScanner::new(start, end); for piece in file.chunks(chunk.max(1)) { scanner.push(piece); } scanner.finish() } #[test] fn locates_and_measures_lines_like_whole_file_decoding() { // BOM, "a", invalid sequence, empty line, a later U+FEFF, and no final newline. let file = [ 0xef, 0xbb, 0xbf, 0x61, 0x0a, 0xe2, 0x82, 0x0a, 0x0a, 0xef, 0xbb, 0xbf, 0x62, 0x0a, 0xc3, 0xa9, ]; for chunk in 1..6 { let all = scan(&file, 0, None, chunk); assert_eq!(all["newlines"], 4); // "a\n\u{fffd}\n\n\u{feff}b\né" is 1+1+3+1+1+3+1+1+2 = 14 bytes. assert_eq!(all["selectedBytes"], 14); assert_eq!(all["firstLineBytes"], 1); let middle = scan(&file, 1, Some(3), chunk); assert_eq!( (middle["start"].as_u64(), middle["end"].as_u64()), (Some(5), Some(8)) ); assert_eq!(middle["selectedBytes"], 4); assert_eq!(middle["lastLineStart"], 8); let past = scan(&file, 9, None, chunk); assert_eq!(past["start"], 16); } } }