diff --git a/src/eval/mod.rs b/src/eval/mod.rs index 22c7c0b43..e911596df 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -23,7 +23,7 @@ pub use value::*; use std::collections::HashMap; use std::io; use std::mem; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::rc::Rc; use crate::diag::{Error, StrResult, Tracepoint, TypResult}; @@ -107,7 +107,7 @@ impl<'a> EvalContext<'a> { /// Process an import of a module relative to the current location. pub fn import(&mut self, path: &str, span: Span) -> TypResult { // Load the source file. - let full = self.relpath(path); + let full = self.make_path(path); let id = self.sources.load(&full).map_err(|err| { Error::boxed(self.source, span, match err.kind() { io::ErrorKind::NotFound => "file not found".into(), @@ -157,15 +157,14 @@ impl<'a> EvalContext<'a> { Ok(id) } - /// Complete a path that is relative to the current file to be relative to - /// the environment's current directory. - pub fn relpath(&self, path: impl AsRef) -> PathBuf { - self.sources - .get(self.source) - .path() - .parent() - .expect("is a file") - .join(path) + /// Complete a user-entered path (relative to the source file) to be + /// relative to the compilation environment's root. + pub fn make_path(&self, path: &str) -> PathBuf { + if let Some(dir) = self.sources.get(self.source).path().parent() { + dir.join(path) + } else { + path.into() + } } } diff --git a/src/library/elements.rs b/src/library/elements.rs index e021c0c63..98165204d 100644 --- a/src/library/elements.rs +++ b/src/library/elements.rs @@ -15,7 +15,7 @@ pub fn image(ctx: &mut EvalContext, args: &mut FuncArgs) -> TypResult { let width = args.named("width")?; let height = args.named("height")?; - let full = ctx.relpath(path.v.as_str()); + let full = ctx.make_path(&path.v); let id = ctx.images.load(&full).map_err(|err| { Error::boxed(args.source, path.span, match err.kind() { io::ErrorKind::NotFound => "file not found".into(), diff --git a/src/parse/mod.rs b/src/parse/mod.rs index f033e01f2..9eabcfc75 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -34,18 +34,16 @@ fn tree(p: &mut Parser) -> SyntaxTree { tree_while(p, true, &mut |_| true) } -/// Parse a syntax tree that stays right of the column at the start of the next -/// non-whitespace token. -fn tree_indented(p: &mut Parser) -> SyntaxTree { +/// Parse a syntax tree that stays right of the given column. +fn tree_indented(p: &mut Parser, column: usize) -> SyntaxTree { p.eat_while(|t| match t { Token::Space(n) => n == 0, Token::LineComment(_) | Token::BlockComment(_) => true, _ => false, }); - let column = p.column(p.next_start()); tree_while(p, false, &mut |p| match p.peek() { - Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) >= column, + Some(Token::Space(n)) if n >= 1 => p.column(p.next_end()) > column, _ => true, }) } @@ -68,7 +66,7 @@ where let start = p.next_start(); let tree = tree_while(p, true, f); call.args.items.push(CallArg::Pos(Expr::Template(TemplateExpr { - span: p.span(start), + span: p.span_from(start), tree: Rc::new(tree), }))); } @@ -189,7 +187,8 @@ fn raw(p: &mut Parser, token: RawToken) -> SyntaxNode { /// Parse a heading. fn heading(p: &mut Parser) -> SyntaxNode { let start = p.next_start(); - p.assert(Token::Eq); + let column = p.column(start); + p.eat_assert(Token::Eq); // Count depth. let mut level: usize = 1; @@ -198,28 +197,29 @@ fn heading(p: &mut Parser) -> SyntaxNode { } if level > 6 { - return SyntaxNode::Text(p.eaten_from(start).into()); + return SyntaxNode::Text(p.get(start .. p.prev_end()).into()); } - let body = tree_indented(p); - - SyntaxNode::Heading(HeadingNode { span: p.span(start), level, body }) + let body = tree_indented(p, column); + SyntaxNode::Heading(HeadingNode { span: p.span_from(start), level, body }) } /// Parse a single list item. fn list_item(p: &mut Parser) -> SyntaxNode { let start = p.next_start(); - p.assert(Token::Hyph); - let body = tree_indented(p); - SyntaxNode::List(ListItem { span: p.span(start), body }) + let column = p.column(start); + p.eat_assert(Token::Hyph); + let body = tree_indented(p, column); + SyntaxNode::List(ListItem { span: p.span_from(start), body }) } /// Parse a single enum item. fn enum_item(p: &mut Parser, number: Option) -> SyntaxNode { let start = p.next_start(); - p.assert(Token::Numbering(number)); - let body = tree_indented(p); - SyntaxNode::Enum(EnumItem { span: p.span(start), number, body }) + let column = p.column(start); + p.eat_assert(Token::Numbering(number)); + let body = tree_indented(p, column); + SyntaxNode::Enum(EnumItem { span: p.span_from(start), number, body }) } /// Parse an expression. @@ -240,7 +240,7 @@ fn expr_with(p: &mut Parser, atomic: bool, min_prec: usize) -> Option { Some(op) => { let prec = op.precedence(); let expr = Box::new(expr_with(p, atomic, prec)?); - Expr::Unary(UnaryExpr { span: p.span(start), op, expr }) + Expr::Unary(UnaryExpr { span: p.span_from(start), op, expr }) } None => primary(p, atomic)?, }; @@ -529,7 +529,7 @@ fn block(p: &mut Parser, scoping: bool) -> Expr { fn call(p: &mut Parser, callee: Expr) -> Option { let mut wide = p.eat_if(Token::Excl); if wide && p.outer_mode() == TokenMode::Code { - let span = p.span(callee.span().start); + let span = p.span_from(callee.span().start); p.error(span, "wide calls are only allowed directly in templates"); wide = false; } @@ -552,7 +552,7 @@ fn call(p: &mut Parser, callee: Expr) -> Option { } Some(Expr::Call(CallExpr { - span: p.span(callee.span().start), + span: p.span_from(callee.span().start), callee: Box::new(callee), wide, args, @@ -571,7 +571,7 @@ fn args(p: &mut Parser) -> CallArgs { fn with_expr(p: &mut Parser, callee: Expr) -> Option { if p.peek() == Some(Token::LeftParen) { Some(Expr::With(WithExpr { - span: p.span(callee.span().start), + span: p.span_from(callee.span().start), callee: Box::new(callee), args: args(p), })) @@ -584,7 +584,7 @@ fn with_expr(p: &mut Parser, callee: Expr) -> Option { /// Parse a let expression. fn let_expr(p: &mut Parser) -> Option { let start = p.next_start(); - p.assert(Token::Let); + p.eat_assert(Token::Let); let mut let_expr = None; if let Some(binding) = ident(p) { @@ -622,7 +622,7 @@ fn let_expr(p: &mut Parser) -> Option { } let_expr = Some(Expr::Let(LetExpr { - span: p.span(start), + span: p.span_from(start), binding, init: init.map(Box::new), })); @@ -634,7 +634,7 @@ fn let_expr(p: &mut Parser) -> Option { /// Parse an if expresion. fn if_expr(p: &mut Parser) -> Option { let start = p.next_start(); - p.assert(Token::If); + p.eat_assert(Token::If); let mut if_expr = None; if let Some(condition) = expr(p) { @@ -650,7 +650,7 @@ fn if_expr(p: &mut Parser) -> Option { } if_expr = Some(Expr::If(IfExpr { - span: p.span(start), + span: p.span_from(start), condition: Box::new(condition), if_body: Box::new(if_body), else_body: else_body.map(Box::new), @@ -664,13 +664,13 @@ fn if_expr(p: &mut Parser) -> Option { /// Parse a while expresion. fn while_expr(p: &mut Parser) -> Option { let start = p.next_start(); - p.assert(Token::While); + p.eat_assert(Token::While); let mut while_expr = None; if let Some(condition) = expr(p) { if let Some(body) = body(p) { while_expr = Some(Expr::While(WhileExpr { - span: p.span(start), + span: p.span_from(start), condition: Box::new(condition), body: Box::new(body), })); @@ -683,15 +683,15 @@ fn while_expr(p: &mut Parser) -> Option { /// Parse a for expression. fn for_expr(p: &mut Parser) -> Option { let start = p.next_start(); - p.assert(Token::For); + p.eat_assert(Token::For); let mut for_expr = None; if let Some(pattern) = for_pattern(p) { - if p.expect(Token::In) { + if p.eat_expect(Token::In) { if let Some(iter) = expr(p) { if let Some(body) = body(p) { for_expr = Some(Expr::For(ForExpr { - span: p.span(start), + span: p.span_from(start), pattern, iter: Box::new(iter), body: Box::new(body), @@ -718,7 +718,7 @@ fn for_pattern(p: &mut Parser) -> Option { /// Parse an import expression. fn import_expr(p: &mut Parser) -> Option { let start = p.next_start(); - p.assert(Token::Import); + p.eat_assert(Token::Import); let imports = if p.eat_if(Token::Star) { // This is the wildcard scenario. @@ -735,10 +735,10 @@ fn import_expr(p: &mut Parser) -> Option { }; let mut import_expr = None; - if p.expect(Token::From) { + if p.eat_expect(Token::From) { if let Some(path) = expr(p) { import_expr = Some(Expr::Import(ImportExpr { - span: p.span(start), + span: p.span_from(start), imports, path: Box::new(path), })); @@ -751,11 +751,11 @@ fn import_expr(p: &mut Parser) -> Option { /// Parse an include expression. fn include_expr(p: &mut Parser) -> Option { let start = p.next_start(); - p.assert(Token::Include); + p.eat_assert(Token::Include); expr(p).map(|path| { Expr::Include(IncludeExpr { - span: p.span(start), + span: p.span_from(start), path: Box::new(path), }) }) diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 326fc280d..4f2e59c60 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -1,5 +1,4 @@ use std::fmt::{self, Debug, Formatter}; -use std::ops::Range; use super::{TokenMode, Tokens}; use crate::diag::Error; @@ -22,9 +21,9 @@ pub struct Parser<'s> { /// (Same as `next` except if we are at the end of group, then `None`). peeked: Option>, /// The end position of the last (non-whitespace if in code mode) token. - prev_end: usize, + prev_end: Pos, /// The start position of the peeked token. - next_start: usize, + next_start: Pos, } /// A logical group of tokens, e.g. `[...]`. @@ -32,7 +31,7 @@ pub struct Parser<'s> { struct GroupEntry { /// The start position of the group. Used by `Parser::end_group` to return /// The group's full span. - pub start: usize, + pub start: Pos, /// The kind of group this is. This decides which tokens will end the group. /// For example, a [`Group::Paren`] will be ended by /// [`Token::RightParen`]. @@ -70,8 +69,8 @@ impl<'s> Parser<'s> { groups: vec![], next, peeked: next, - prev_end: 0, - next_start: 0, + prev_end: Pos::ZERO, + next_start: Pos::ZERO, } } @@ -80,37 +79,144 @@ impl<'s> Parser<'s> { self.errors } - /// Add an error with location and message. - pub fn error(&mut self, span: impl Into, message: impl Into) { - self.errors.push(Error::new(self.source.id(), span, message)); + /// Whether the end of the source string or group is reached. + pub fn eof(&self) -> bool { + self.peek().is_none() } - /// Eat the next token and add an error that it is not the expected `thing`. - pub fn expected(&mut self, what: &str) { - let before = self.next_start(); - if let Some(found) = self.eat() { - let after = self.prev_end(); - self.error( - before .. after, - format!("expected {}, found {}", what, found.name()), - ); + /// Consume the next token. + pub fn eat(&mut self) -> Option> { + let token = self.peek()?; + self.bump(); + Some(token) + } + + /// Eat the next token and return its source range. + pub fn eat_span(&mut self) -> Span { + let start = self.next_start(); + self.eat(); + Span::new(start, self.prev_end()) + } + + /// Consume the next token if it is the given one. + pub fn eat_if(&mut self, t: Token) -> bool { + if self.peek() == Some(t) { + self.bump(); + true } else { - self.expected_at(self.next_start(), what); + false } } - /// Add an error that `what` was expected at the given position. - pub fn expected_at(&mut self, pos: impl Into, what: &str) { - self.error(pos.into(), format!("expected {}", what)); + /// Consume the next token if the closure maps it a to `Some`-variant. + pub fn eat_map(&mut self, f: F) -> Option + where + F: FnOnce(Token<'s>) -> Option, + { + let token = self.peek()?; + let mapped = f(token); + if mapped.is_some() { + self.bump(); + } + mapped } - /// Eat the next token and add an error that it is unexpected. - pub fn unexpected(&mut self) { - let before = self.next_start(); - if let Some(found) = self.eat() { - let after = self.prev_end(); - self.error(before .. after, format!("unexpected {}", found.name())); + /// Consume the next token if it is the given one and produce an error if + /// not. + pub fn eat_expect(&mut self, t: Token) -> bool { + let eaten = self.eat_if(t); + if !eaten { + self.expected_at(self.prev_end(), t.name()); } + eaten + } + + /// Consume the next token, debug-asserting that it is one of the given ones. + pub fn eat_assert(&mut self, t: Token) { + let next = self.eat(); + debug_assert_eq!(next, Some(t)); + } + + /// Consume tokens while the condition is true. + pub fn eat_while(&mut self, mut f: F) + where + F: FnMut(Token<'s>) -> bool, + { + while self.peek().map_or(false, |t| f(t)) { + self.eat(); + } + } + + /// Peek at the next token without consuming it. + pub fn peek(&self) -> Option> { + self.peeked + } + + /// Peek at the next token if it follows immediately after the last one + /// without any whitespace in between. + pub fn peek_direct(&self) -> Option> { + if self.next_start() == self.prev_end() { + self.peeked + } else { + None + } + } + + /// Peek at the span of the next token. + /// + /// Has length zero if `peek()` returns `None`. + pub fn peek_span(&self) -> Span { + Span::new(self.next_start(), self.next_end()) + } + + /// Peek at the source of the next token. + pub fn peek_src(&self) -> &'s str { + self.get(self.peek_span()) + } + + /// Checks whether the next token fulfills a condition. + /// + /// Returns `false` if there is no next token. + pub fn check(&self, f: F) -> bool + where + F: FnOnce(Token<'s>) -> bool, + { + self.peek().map_or(false, f) + } + + /// The byte position at which the last token ended. + /// + /// Refers to the end of the last _non-whitespace_ token in code mode. + pub fn prev_end(&self) -> Pos { + self.prev_end.into() + } + + /// The byte position at which the next token starts. + pub fn next_start(&self) -> Pos { + self.next_start.into() + } + + /// The byte position at which the next token will end. + /// + /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns + /// `None`. + pub fn next_end(&self) -> Pos { + self.tokens.index().into() + } + + /// The span from `start` to [`self.prev_end()`](Self::prev_end). + pub fn span_from(&self, start: Pos) -> Span { + Span::new(start, self.prev_end()) + } + + /// Determine the column index for the given byte position. + pub fn column(&self, pos: Pos) -> usize { + self.source.pos_to_column(pos).unwrap() + } + + /// Slice out part of the source string. + pub fn get(&self, span: impl Into) -> &'s str { + self.tokens.scanner().get(span.into().to_range()) } /// Continue parsing in a group. @@ -131,9 +237,9 @@ impl<'s> Parser<'s> { self.repeek(); match kind { - Group::Paren => self.assert(Token::LeftParen), - Group::Bracket => self.assert(Token::LeftBracket), - Group::Brace => self.assert(Token::LeftBrace), + Group::Paren => self.eat_assert(Token::LeftParen), + Group::Bracket => self.eat_assert(Token::LeftBracket), + Group::Brace => self.eat_assert(Token::LeftBrace), Group::Stmt => {} Group::Expr => {} Group::Imports => {} @@ -171,7 +277,8 @@ impl<'s> Parser<'s> { // Rescan the peeked token if the mode changed. if rescan { - self.jump(self.prev_end()); + self.tokens.jump(self.prev_end().to_usize()); + self.bump(); } Span::new(group.start, self.prev_end()) @@ -188,163 +295,43 @@ impl<'s> Parser<'s> { self.groups.last().map_or(TokenMode::Markup, |group| group.outer_mode) } - /// Whether the end of the source string or group is reached. - pub fn eof(&self) -> bool { - self.peek().is_none() + /// Add an error with location and message. + pub fn error(&mut self, span: impl Into, message: impl Into) { + self.errors.push(Error::new(self.source.id(), span, message)); } - /// Peek at the next token without consuming it. - pub fn peek(&self) -> Option> { - self.peeked - } - - /// Peek at the next token if it follows immediately after the last one - /// without any whitespace in between. - pub fn peek_direct(&self) -> Option> { - if self.next_start() == self.prev_end() { - self.peeked + /// Eat the next token and add an error that it is not the expected `thing`. + pub fn expected(&mut self, what: &str) { + let before = self.next_start(); + if let Some(found) = self.eat() { + let after = self.prev_end(); + self.error( + before .. after, + format!("expected {}, found {}", what, found.name()), + ); } else { - None + self.expected_at(self.next_start(), what); } } - /// Peek at the span of the next token. - /// - /// Has length zero if `peek()` returns `None`. - pub fn peek_span(&self) -> Span { - self.peek_range().into() + /// Add an error that `what` was expected at the given position. + pub fn expected_at(&mut self, pos: Pos, what: &str) { + self.error(pos, format!("expected {}", what)); } - /// Peek at the source of the next token. - pub fn peek_src(&self) -> &'s str { - self.tokens.scanner().get(self.peek_range()) - } - - /// Peek at the source range (start and end index) of the next token. - pub fn peek_range(&self) -> Range { - self.next_start() .. self.next_end() - } - - /// Checks whether the next token fulfills a condition. - /// - /// Returns `false` if there is no next token. - pub fn check(&self, f: F) -> bool - where - F: FnOnce(Token<'s>) -> bool, - { - self.peek().map_or(false, f) - } - - /// Consume the next token. - pub fn eat(&mut self) -> Option> { - let token = self.peek()?; - self.bump(); - Some(token) - } - - /// Consume the next token if it is the given one. - pub fn eat_if(&mut self, t: Token) -> bool { - if self.peek() == Some(t) { - self.bump(); - true - } else { - false + /// Eat the next token and add an error that it is unexpected. + pub fn unexpected(&mut self) { + let before = self.next_start(); + if let Some(found) = self.eat() { + let after = self.prev_end(); + self.error(before .. after, format!("unexpected {}", found.name())); } } - /// Consume tokens while the condition is true. - pub fn eat_while(&mut self, mut f: F) - where - F: FnMut(Token<'s>) -> bool, - { - while self.peek().map_or(false, |t| f(t)) { - self.eat(); - } - } - - /// Consume the next token if the closure maps it a to `Some`-variant. - pub fn eat_map(&mut self, f: F) -> Option - where - F: FnOnce(Token<'s>) -> Option, - { - let token = self.peek()?; - let mapped = f(token); - if mapped.is_some() { - self.bump(); - } - mapped - } - - /// Eat the next token and return its source range. - pub fn eat_span(&mut self) -> Span { - let start = self.next_start(); - self.eat(); - Span::new(start, self.prev_end()) - } - - /// Consume the next token if it is the given one and produce an error if - /// not. - pub fn expect(&mut self, t: Token) -> bool { - let eaten = self.eat_if(t); - if !eaten { - self.expected_at(self.prev_end(), t.name()); - } - eaten - } - - /// Consume the next token, debug-asserting that it is one of the given ones. - pub fn assert(&mut self, t: Token) { - let next = self.eat(); - debug_assert_eq!(next, Some(t)); - } - - /// The index at which the last token ended. - /// - /// Refers to the end of the last _non-whitespace_ token in code mode. - pub fn prev_end(&self) -> usize { - self.prev_end - } - - /// The index at which the next token starts. - pub fn next_start(&self) -> usize { - self.next_start - } - - /// The index at which the next token will end. - /// - /// Is the same as [`next_start()`][Self::next_start] if `peek()` returns - /// `None`. - pub fn next_end(&self) -> usize { - self.tokens.index() - } - - /// Determine the column for the given index in the source. - pub fn column(&self, index: usize) -> usize { - self.source.pos_to_column(index.into()).unwrap() - } - - /// The span from `start` to [`self.prev_end()`](Self::prev_end). - pub fn span(&self, start: impl Into) -> Span { - Span::new(start, self.prev_end()) - } - - /// Return the source string from `start` to the end of the previous token. - pub fn eaten_from(&self, start: usize) -> &'s str { - self.tokens.scanner().get(start .. self.prev_end()) - } - - /// Jump to an index in the string. - /// - /// You need to know the correct column. - fn jump(&mut self, index: usize) { - self.tokens.jump(index); - self.bump(); - } - /// Move to the next token. fn bump(&mut self) { - self.prev_end = self.tokens.index(); - self.next_start = self.tokens.index(); + self.prev_end = self.tokens.index().into(); + self.next_start = self.tokens.index().into(); self.next = self.tokens.next(); if self.tokens.mode() == TokenMode::Code { @@ -355,7 +342,7 @@ impl<'s> Parser<'s> { Some(Token::BlockComment(_)) => true, _ => false, } { - self.next_start = self.tokens.index(); + self.next_start = self.tokens.index().into(); self.next = self.tokens.next(); } } @@ -399,7 +386,7 @@ impl<'s> Parser<'s> { impl Debug for Parser<'_> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { let mut s = self.tokens.scanner(); - s.jump(self.next_start()); + s.jump(self.next_start().to_usize()); write!(f, "Parser({}|{})", s.eaten(), s.rest()) } } diff --git a/src/source.rs b/src/source.rs index 20ba137f2..0a5a28609 100644 --- a/src/source.rs +++ b/src/source.rs @@ -63,7 +63,7 @@ impl SourceStore { io::Error::new(io::ErrorKind::InvalidData, "file is not valid utf-8") })?; - Ok(self.insert(Some(hash), path, src)) + Ok(self.insert(path, src, Some(hash))) } /// Directly provide a source file. @@ -82,16 +82,16 @@ impl SourceStore { id } else { // Not loaded yet. - self.insert(Some(hash), path, src) + self.insert(path, src, Some(hash)) } } else { // Not known to the loader. - self.insert(None, path, src) + self.insert(path, src, None) } } /// Insert a new source file. - fn insert(&mut self, hash: Option, path: &Path, src: String) -> SourceId { + fn insert(&mut self, path: &Path, src: String, hash: Option) -> SourceId { let id = SourceId(self.sources.len() as u32); if let Some(hash) = hash { self.files.insert(hash, id); @@ -112,6 +112,9 @@ impl SourceStore { } /// A single source file. +/// +/// _Note_: All line and column indices start at zero, just like byte indices. +/// Only for user-facing display, you should add 1 to them. pub struct SourceFile { id: SourceId, path: PathBuf, @@ -120,7 +123,8 @@ pub struct SourceFile { } impl SourceFile { - fn new(id: SourceId, path: &Path, src: String) -> Self { + /// Create a new source file. + pub fn new(id: SourceId, path: &Path, src: String) -> Self { let mut line_starts = vec![Pos::ZERO]; let mut s = Scanner::new(&src); @@ -151,7 +155,7 @@ impl SourceFile { self.id } - /// The path to the source file. + /// The normalized path to the source file. pub fn path(&self) -> &Path { &self.path } @@ -161,6 +165,11 @@ impl SourceFile { &self.src } + /// Slice out the part of the source code enclosed by the span. + pub fn get(&self, span: impl Into) -> Option<&str> { + self.src.get(span.into().to_range()) + } + /// Get the length of the file in bytes. pub fn len_bytes(&self) -> usize { self.src.len() @@ -171,11 +180,6 @@ impl SourceFile { self.line_starts.len() } - /// Slice out the part of the source code enclosed by the span. - pub fn get(&self, span: Span) -> Option<&str> { - self.src.get(span.to_range()) - } - /// Return the index of the line that contains the given byte position. pub fn pos_to_line(&self, byte_pos: Pos) -> Option { (byte_pos.to_usize() <= self.src.len()).then(|| { @@ -186,14 +190,15 @@ impl SourceFile { }) } - /// Return the column of the byte index. + /// Return the index of the column at the byte index. /// - /// Tabs are counted as occupying two columns. + /// The column is defined as the number of characters in the line before the + /// byte position. pub fn pos_to_column(&self, byte_pos: Pos) -> Option { let line = self.pos_to_line(byte_pos)?; let start = self.line_to_pos(line)?; let head = self.get(Span::new(start, byte_pos))?; - Some(head.chars().map(width).sum()) + Some(head.chars().count()) } /// Return the byte position at which the given line starts. @@ -210,32 +215,19 @@ impl SourceFile { /// Return the byte position of the given (line, column) pair. /// - /// Tabs are counted as occupying two columns. + /// The column defines the number of characters to go beyond the start of + /// the line. pub fn line_column_to_pos(&self, line_idx: usize, column_idx: usize) -> Option { let span = self.line_to_span(line_idx)?; let line = self.get(span)?; - - if column_idx == 0 { - return Some(span.start); + let mut chars = line.chars(); + for _ in 0 .. column_idx { + chars.next(); } - - let mut column = 0; - for (i, c) in line.char_indices() { - column += width(c); - if column >= column_idx { - return Some(span.start + Pos::from(i + c.len_utf8())); - } - } - - None + Some(span.start + (line.len() - chars.as_str().len())) } } -/// The display width of the character. -fn width(c: char) -> usize { - if c == '\t' { 2 } else { 1 } -} - impl AsRef for SourceFile { fn as_ref(&self) -> &str { &self.src @@ -256,14 +248,34 @@ impl<'a> Files<'a> for SourceStore { Ok(self.get(id)) } - fn line_index( + fn line_index(&'a self, id: SourceId, given: usize) -> Result { + let source = self.get(id); + source + .pos_to_line(given.into()) + .ok_or_else(|| files::Error::IndexTooLarge { given, max: source.len_bytes() }) + } + + fn line_range( &'a self, id: SourceId, - byte_index: usize, + given: usize, + ) -> Result, files::Error> { + let source = self.get(id); + source + .line_to_span(given) + .map(Span::to_range) + .ok_or_else(|| files::Error::LineTooLarge { given, max: source.len_lines() }) + } + + fn column_number( + &'a self, + id: SourceId, + _: usize, + given: usize, ) -> Result { let source = self.get(id); - source.pos_to_line(byte_index.into()).ok_or_else(|| { - let (given, max) = (byte_index, source.len_bytes()); + source.pos_to_column(given.into()).ok_or_else(|| { + let max = source.len_bytes(); if given <= max { files::Error::InvalidCharBoundary { given } } else { @@ -271,28 +283,13 @@ impl<'a> Files<'a> for SourceStore { } }) } - - fn line_range( - &'a self, - id: SourceId, - line_index: usize, - ) -> Result, files::Error> { - let source = self.get(id); - match source.line_to_span(line_index) { - Some(span) => Ok(span.to_range()), - None => Err(files::Error::LineTooLarge { - given: line_index, - max: source.len_lines(), - }), - } - } } #[cfg(test)] mod tests { use super::*; - const TEST: &str = "äbcde\nf💛g\r\nhi\rjkl"; + const TEST: &str = "ä\tcde\nf💛g\r\nhi\rjkl"; #[test] fn test_source_file_new() { @@ -313,6 +310,17 @@ mod tests { assert_eq!(source.pos_to_line(Pos(22)), None); } + #[test] + fn test_source_file_pos_to_column() { + let source = SourceFile::detached(TEST); + assert_eq!(source.pos_to_column(Pos(0)), Some(0)); + assert_eq!(source.pos_to_column(Pos(2)), Some(1)); + assert_eq!(source.pos_to_column(Pos(6)), Some(5)); + assert_eq!(source.pos_to_column(Pos(7)), Some(0)); + assert_eq!(source.pos_to_column(Pos(8)), Some(1)); + assert_eq!(source.pos_to_column(Pos(12)), Some(2)); + } + #[test] fn test_source_file_roundtrip() { #[track_caller] diff --git a/tests/ref/markup/lists.png b/tests/ref/markup/lists.png index 815e51056..315905b8e 100644 Binary files a/tests/ref/markup/lists.png and b/tests/ref/markup/lists.png differ diff --git a/tests/typ/markup/enums.typ b/tests/typ/markup/enums.typ index 516fd0c1c..ecf08a36a 100644 --- a/tests/typ/markup/enums.typ +++ b/tests/typ/markup/enums.typ @@ -8,4 +8,4 @@ --- 1. First. 2. Second. - 1. Back to first. +1. Back to first. diff --git a/tests/typ/markup/heading.typ b/tests/typ/markup/heading.typ index bc0317b37..4647e7a6e 100644 --- a/tests/typ/markup/heading.typ +++ b/tests/typ/markup/heading.typ @@ -33,7 +33,7 @@ No = heading indented. = This - is not. +is not. // Code blocks continue heading. = A { diff --git a/tests/typ/markup/lists.typ b/tests/typ/markup/lists.typ index 5c464ab8b..716fcd739 100644 --- a/tests/typ/markup/lists.typ +++ b/tests/typ/markup/lists.typ @@ -6,6 +6,12 @@ _Shopping list_ - Potatoes - Juice +--- +Tightly +- surrounded +- by two +paragraphs. + --- - First level. - Second level. @@ -18,28 +24,26 @@ _Shopping list_ - At the top. --- -- Works - - Also with four spaces - - Or two tabs +- Level 1 + - Level [ +2 through template +] --- - Top-level indent - is fine. ---- -Tightly -- surrounded -- by two -paragraphs. - --- - A - B - C - - D +- D --- -- Level 1 - - Level [ -2 through template -] +// This works because tabs are used consistently. + - A with 1 tab + - B with 2 tabs + +// This doesn't work because of mixed tabs and spaces. + - A with 2 spaces + - B with 2 tabs