//! Source file management. use std::collections::HashMap; use std::io; use std::ops::Range; use std::path::{Path, PathBuf}; use std::rc::Rc; use crate::diag::TypResult; use crate::loading::{FileHash, Loader}; use crate::parse::{is_newline, parse, Reparser, Scanner}; use crate::syntax::ast::Markup; use crate::syntax::{self, Category, GreenNode, RedNode}; use crate::util::{PathExt, StrExt}; #[cfg(feature = "codespan-reporting")] use codespan_reporting::files::{self, Files}; /// A unique identifier for a loaded source file. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub struct SourceId(u32); impl SourceId { /// Create a source id from the raw underlying value. /// /// This should only be called with values returned by /// [`into_raw`](Self::into_raw). pub const fn from_raw(v: u32) -> Self { Self(v) } /// Convert into the raw underlying value. pub const fn into_raw(self) -> u32 { self.0 } } /// Storage for loaded source files. pub struct SourceStore { loader: Rc, files: HashMap, sources: Vec, } impl SourceStore { /// Create a new, empty source store. pub fn new(loader: Rc) -> Self { Self { loader, files: HashMap::new(), sources: vec![], } } /// Load a source file from a path using the `loader`. pub fn load(&mut self, path: &Path) -> io::Result { let hash = self.loader.resolve(path)?; if let Some(&id) = self.files.get(&hash) { return Ok(id); } let data = self.loader.load(path)?; let src = String::from_utf8(data).map_err(|_| { io::Error::new(io::ErrorKind::InvalidData, "file is not valid utf-8") })?; Ok(self.provide(path, src)) } /// Directly provide a source file. /// /// The `path` does not need to be [resolvable](Loader::resolve) through the /// `loader`. If it is though, imports that resolve to the same file hash /// will use the inserted file instead of going through [`Loader::load`]. /// /// If the path is resolvable and points to an existing source file, it is /// overwritten. pub fn provide(&mut self, path: &Path, src: String) -> SourceId { let hash = self.loader.resolve(path).ok(); // Check for existing file and replace if one exists. if let Some(&id) = hash.and_then(|hash| self.files.get(&hash)) { self.sources[id.0 as usize] = SourceFile::new(id, path, src); return id; } // No existing file yet. let id = SourceId(self.sources.len() as u32); self.sources.push(SourceFile::new(id, path, src)); // Register in file map if the path was known to the loader. if let Some(hash) = hash { self.files.insert(hash, id); } id } /// Get a reference to a loaded source file. /// /// This panics if no source file with this `id` exists. This function /// should only be called with ids returned by this store's /// [`load()`](Self::load) and [`provide()`](Self::provide) methods. #[track_caller] pub fn get(&self, id: SourceId) -> &SourceFile { &self.sources[id.0 as usize] } /// Edit a source file by replacing the given range. /// /// This panics if no source file with this `id` exists or if the `replace` /// range is out of bounds for the source file identified by `id`. #[track_caller] pub fn edit(&mut self, id: SourceId, replace: Range, with: &str) { self.sources[id.0 as usize].edit(replace, with); } } /// A single source file. /// /// _Note_: All line and column indices start at zero, just like byte indices. /// Only for user-facing display, you should add 1 to them. pub struct SourceFile { id: SourceId, path: PathBuf, src: String, lines: Vec, root: Rc, } impl SourceFile { /// Create a new source file. pub fn new(id: SourceId, path: &Path, src: String) -> Self { let mut lines = vec![Line { byte_idx: 0, utf16_idx: 0 }]; lines.extend(Line::iter(0, 0, &src)); Self { id, path: path.normalize(), root: parse(&src), src, lines, } } /// Create a source file without a real id and path, usually for testing. pub fn detached(src: impl Into) -> Self { Self::new(SourceId(0), Path::new(""), src.into()) } /// The root node of the file's untyped green tree. pub fn root(&self) -> &Rc { &self.root } /// The root red node of the file's untyped red tree. pub fn red(&self) -> RedNode { RedNode::from_root(self.root.clone(), self.id) } /// The root node of the file's typed abstract syntax tree. pub fn ast(&self) -> TypResult { let red = self.red(); let errors = red.errors(); if errors.is_empty() { Ok(red.cast().unwrap()) } else { Err(Box::new(errors)) } } /// The id of the source file. pub fn id(&self) -> SourceId { self.id } /// The normalized path to the source file. pub fn path(&self) -> &Path { &self.path } /// The whole source as a string slice. pub fn src(&self) -> &str { &self.src } /// Slice out the part of the source code enclosed by the span. pub fn get(&self, range: Range) -> Option<&str> { self.src.get(range) } /// Get the length of the file in bytes. pub fn len_bytes(&self) -> usize { self.src.len() } /// Get the length of the file in UTF16 code units. pub fn len_utf16(&self) -> usize { let last = self.lines.last().unwrap(); last.utf16_idx + self.src[last.byte_idx ..].len_utf16() } /// Get the length of the file in lines. pub fn len_lines(&self) -> usize { self.lines.len() } /// Return the index of the UTF-16 code unit at the byte index. pub fn byte_to_utf16(&self, byte_idx: usize) -> Option { let line_idx = self.byte_to_line(byte_idx)?; let line = self.lines.get(line_idx)?; let head = self.src.get(line.byte_idx .. byte_idx)?; Some(line.utf16_idx + head.len_utf16()) } /// Return the index of the line that contains the given byte index. pub fn byte_to_line(&self, byte_idx: usize) -> Option { (byte_idx <= self.src.len()).then(|| { match self.lines.binary_search_by_key(&byte_idx, |line| line.byte_idx) { Ok(i) => i, Err(i) => i - 1, } }) } /// Return the index of the column at the byte index. /// /// The column is defined as the number of characters in the line before the /// byte index. pub fn byte_to_column(&self, byte_idx: usize) -> Option { let line = self.byte_to_line(byte_idx)?; let start = self.line_to_byte(line)?; let head = self.get(start .. byte_idx)?; Some(head.chars().count()) } /// Return the byte index at the UTF-16 code unit. pub fn utf16_to_byte(&self, utf16_idx: usize) -> Option { let line = self.lines.get( match self.lines.binary_search_by_key(&utf16_idx, |line| line.utf16_idx) { Ok(i) => i, Err(i) => i - 1, }, )?; let mut k = line.utf16_idx; for (i, c) in self.src[line.byte_idx ..].char_indices() { if k >= utf16_idx { return Some(line.byte_idx + i); } k += c.len_utf16(); } (k == utf16_idx).then(|| self.src.len()) } /// Return the byte position at which the given line starts. pub fn line_to_byte(&self, line_idx: usize) -> Option { self.lines.get(line_idx).map(|line| line.byte_idx) } /// Return the range which encloses the given line. pub fn line_to_range(&self, line_idx: usize) -> Option> { let start = self.line_to_byte(line_idx)?; let end = self.line_to_byte(line_idx + 1).unwrap_or(self.src.len()); Some(start .. end) } /// Return the byte index of the given (line, column) pair. /// /// The column defines the number of characters to go beyond the start of /// the line. pub fn line_column_to_byte( &self, line_idx: usize, column_idx: usize, ) -> Option { let range = self.line_to_range(line_idx)?; let line = self.get(range.clone())?; let mut chars = line.chars(); for _ in 0 .. column_idx { chars.next(); } Some(range.start + (line.len() - chars.as_str().len())) } /// Edit the source file by replacing the given range. /// /// Returns the range of the section in the new source that was ultimately /// reparsed. The method panics if the `replace` range is out of bounds. pub fn edit(&mut self, replace: Range, with: &str) -> Range { let start_byte = replace.start; let start_utf16 = self.byte_to_utf16(replace.start).unwrap(); self.src.replace_range(replace.clone(), with); // Remove invalidated line starts. let line = self.byte_to_line(start_byte).unwrap(); self.lines.truncate(line + 1); // Handle adjoining of \r and \n. if self.src[.. start_byte].ends_with('\r') && with.starts_with('\n') { self.lines.pop(); } // Recalculate the line starts after the edit. self.lines.extend(Line::iter( start_byte, start_utf16, &self.src[start_byte ..], )); // Incrementally reparse the replaced range. Reparser::new(&self.src, replace, with.len()).reparse(&mut self.root) } /// Provide highlighting categories for the given range of the source file. pub fn highlight(&self, range: Range, mut f: F) where F: FnMut(Range, Category), { syntax::highlight(self.red().as_ref(), range, &mut f) } } /// Metadata about a line. #[derive(Debug, Copy, Clone, Eq, PartialEq)] struct Line { /// The UTF-8 byte offset where the line starts. byte_idx: usize, /// The UTF-16 codepoint offset where the line starts. utf16_idx: usize, } impl Line { /// Iterate over the lines in the string. fn iter( byte_offset: usize, utf16_offset: usize, string: &str, ) -> impl Iterator + '_ { let mut s = Scanner::new(string); let mut utf16_idx = utf16_offset; std::iter::from_fn(move || { s.eat_until(|c| { utf16_idx += c.len_utf16(); is_newline(c) }); if s.eof() { return None; } if s.eat() == Some('\r') && s.eat_if('\n') { utf16_idx += 1; } Some(Line { byte_idx: byte_offset + s.index(), utf16_idx, }) }) } } impl AsRef for SourceFile { fn as_ref(&self) -> &str { &self.src } } #[cfg(feature = "codespan-reporting")] impl<'a> Files<'a> for SourceStore { type FileId = SourceId; type Name = std::path::Display<'a>; type Source = &'a SourceFile; fn name(&'a self, id: SourceId) -> Result { Ok(self.get(id).path().display()) } fn source(&'a self, id: SourceId) -> Result { Ok(self.get(id)) } fn line_index(&'a self, id: SourceId, given: usize) -> Result { let source = self.get(id); source .byte_to_line(given) .ok_or_else(|| files::Error::IndexTooLarge { given, max: source.len_bytes() }) } fn line_range( &'a self, id: SourceId, given: usize, ) -> Result, files::Error> { let source = self.get(id); source .line_to_range(given) .ok_or_else(|| files::Error::LineTooLarge { given, max: source.len_lines() }) } fn column_number( &'a self, id: SourceId, _: usize, given: usize, ) -> Result { let source = self.get(id); source.byte_to_column(given).ok_or_else(|| { let max = source.len_bytes(); if given <= max { files::Error::InvalidCharBoundary { given } } else { files::Error::IndexTooLarge { given, max } } }) } } #[cfg(test)] mod tests { use super::*; const TEST: &str = "ä\tcde\nf💛g\r\nhi\rjkl"; #[test] fn test_source_file_new() { let source = SourceFile::detached(TEST); assert_eq!(source.lines, [ Line { byte_idx: 0, utf16_idx: 0 }, Line { byte_idx: 7, utf16_idx: 6 }, Line { byte_idx: 15, utf16_idx: 12 }, Line { byte_idx: 18, utf16_idx: 15 }, ]); } #[test] fn test_source_file_pos_to_line() { let source = SourceFile::detached(TEST); assert_eq!(source.byte_to_line(0), Some(0)); assert_eq!(source.byte_to_line(2), Some(0)); assert_eq!(source.byte_to_line(6), Some(0)); assert_eq!(source.byte_to_line(7), Some(1)); assert_eq!(source.byte_to_line(8), Some(1)); assert_eq!(source.byte_to_line(12), Some(1)); assert_eq!(source.byte_to_line(21), Some(3)); assert_eq!(source.byte_to_line(22), None); } #[test] fn test_source_file_pos_to_column() { let source = SourceFile::detached(TEST); assert_eq!(source.byte_to_column(0), Some(0)); assert_eq!(source.byte_to_column(2), Some(1)); assert_eq!(source.byte_to_column(6), Some(5)); assert_eq!(source.byte_to_column(7), Some(0)); assert_eq!(source.byte_to_column(8), Some(1)); assert_eq!(source.byte_to_column(12), Some(2)); } #[test] fn test_source_file_utf16() { #[track_caller] fn roundtrip(source: &SourceFile, byte_idx: usize, utf16_idx: usize) { let middle = source.byte_to_utf16(byte_idx).unwrap(); let result = source.utf16_to_byte(middle).unwrap(); assert_eq!(middle, utf16_idx); assert_eq!(result, byte_idx); } let source = SourceFile::detached(TEST); roundtrip(&source, 0, 0); roundtrip(&source, 2, 1); roundtrip(&source, 3, 2); roundtrip(&source, 8, 7); roundtrip(&source, 12, 9); roundtrip(&source, 21, 18); assert_eq!(source.byte_to_utf16(22), None); assert_eq!(source.utf16_to_byte(19), None); } #[test] fn test_source_file_roundtrip() { #[track_caller] fn roundtrip(source: &SourceFile, byte_idx: usize) { let line = source.byte_to_line(byte_idx).unwrap(); let column = source.byte_to_column(byte_idx).unwrap(); let result = source.line_column_to_byte(line, column).unwrap(); assert_eq!(result, byte_idx); } let source = SourceFile::detached(TEST); roundtrip(&source, 0); roundtrip(&source, 7); roundtrip(&source, 12); roundtrip(&source, 21); } #[test] fn test_source_file_edit() { #[track_caller] fn test(prev: &str, range: Range, with: &str, after: &str) { let mut source = SourceFile::detached(prev); let result = SourceFile::detached(after); source.edit(range, with); assert_eq!(source.src, result.src); assert_eq!(source.lines, result.lines); } // Test inserting at the begining. test("abc\n", 0 .. 0, "hi\n", "hi\nabc\n"); test("\nabc", 0 .. 0, "hi\r", "hi\r\nabc"); // Test editing in the middle. test(TEST, 4 .. 16, "❌", "ä\tc❌i\rjkl"); // Test appending. test("abc\ndef", 7 .. 7, "hi", "abc\ndefhi"); test("abc\ndef\n", 8 .. 8, "hi", "abc\ndef\nhi"); // Test appending with adjoining \r and \n. test("abc\ndef\r", 8 .. 8, "\nghi", "abc\ndef\r\nghi"); // Test removing everything. test(TEST, 0 .. 21, "", ""); } }