From ad4ef68a112dedabf80f885a02bcb574eb9af9e4 Mon Sep 17 00:00:00 2001 From: astrale-sharp Date: Wed, 12 Jun 2024 14:00:22 +0200 Subject: [PATCH] Lexer hint (#4346) Co-authored-by: Laurenz --- crates/typst-syntax/src/lexer.rs | 22 +++++++++++++++++----- crates/typst-syntax/src/node.rs | 26 ++++++++++++++------------ crates/typst-syntax/src/parser.rs | 11 +++++++---- tests/suite/foundations/array.typ | 1 + tests/suite/scripting/call.typ | 1 + tests/suite/syntax/comment.typ | 1 + 6 files changed, 41 insertions(+), 21 deletions(-) diff --git a/crates/typst-syntax/src/lexer.rs b/crates/typst-syntax/src/lexer.rs index 3f409b2d0..dd05e73f2 100644 --- a/crates/typst-syntax/src/lexer.rs +++ b/crates/typst-syntax/src/lexer.rs @@ -4,7 +4,7 @@ use unicode_script::{Script, UnicodeScript}; use unicode_segmentation::UnicodeSegmentation; use unscanny::Scanner; -use crate::SyntaxKind; +use crate::{SyntaxError, SyntaxKind}; /// Splits up a string of source code into tokens. #[derive(Clone)] @@ -19,7 +19,7 @@ pub(super) struct Lexer<'s> { /// The state held by raw line lexing. raw: Vec<(SyntaxKind, usize)>, /// An error for the last token. - error: Option, + error: Option, } /// What kind of tokens to emit. @@ -75,7 +75,7 @@ impl<'s> Lexer<'s> { } /// Take out the last error, if any. - pub fn take_error(&mut self) -> Option { + pub fn take_error(&mut self) -> Option { self.error.take() } } @@ -83,9 +83,16 @@ impl<'s> Lexer<'s> { impl Lexer<'_> { /// Construct a full-positioned syntax error. fn error(&mut self, message: impl Into) -> SyntaxKind { - self.error = Some(message.into()); + self.error = Some(SyntaxError::new(message)); SyntaxKind::Error } + + /// If the current node is an error, adds a hint. + fn hint(&mut self, message: impl Into) { + if let Some(error) = &mut self.error { + error.hints.push(message.into()); + } + } } /// Shared methods with all [`LexMode`]. @@ -109,7 +116,12 @@ impl Lexer<'_> { Some('/') if self.s.eat_if('/') => self.line_comment(), Some('/') if self.s.eat_if('*') => self.block_comment(), Some('*') if self.s.eat_if('/') => { - self.error("unexpected end of block comment") + let kind = self.error("unexpected end of block comment"); + self.hint( + "consider escaping the `*` with a backslash or \ + opening the block comment with `/*`", + ); + kind } Some(c) => match self.mode { diff --git a/crates/typst-syntax/src/node.rs b/crates/typst-syntax/src/node.rs index 8e623d51e..bc378e667 100644 --- a/crates/typst-syntax/src/node.rs +++ b/crates/typst-syntax/src/node.rs @@ -35,8 +35,8 @@ impl SyntaxNode { } /// Create a new error node. - pub fn error(message: impl Into, text: impl Into) -> Self { - Self(Repr::Error(Arc::new(ErrorNode::new(message, text)))) + pub fn error(error: SyntaxError, text: impl Into) -> Self { + Self(Repr::Error(Arc::new(ErrorNode::new(error, text)))) } /// Create a dummy node of the given kind. @@ -209,7 +209,7 @@ impl SyntaxNode { pub(super) fn convert_to_error(&mut self, message: impl Into) { if !self.kind().is_error() { let text = std::mem::take(self).into_text(); - *self = SyntaxNode::error(message, text); + *self = SyntaxNode::error(SyntaxError::new(message), text); } } @@ -628,15 +628,8 @@ struct ErrorNode { impl ErrorNode { /// Create new error node. - fn new(message: impl Into, text: impl Into) -> Self { - Self { - text: text.into(), - error: SyntaxError { - span: Span::detached(), - message: message.into(), - hints: eco_vec![], - }, - } + fn new(error: SyntaxError, text: impl Into) -> Self { + Self { text: text.into(), error } } /// The byte length of the node in the source text. @@ -674,6 +667,15 @@ pub struct SyntaxError { } impl SyntaxError { + /// Create a new detached syntax error. + pub fn new(message: impl Into) -> Self { + Self { + span: Span::detached(), + message: message.into(), + hints: eco_vec![], + } + } + /// Whether the two errors are the same apart from spans. fn spanless_eq(&self, other: &Self) -> bool { self.message == other.message && self.hints == other.hints diff --git a/crates/typst-syntax/src/parser.rs b/crates/typst-syntax/src/parser.rs index e27cafcfd..7e7eeea5f 100644 --- a/crates/typst-syntax/src/parser.rs +++ b/crates/typst-syntax/src/parser.rs @@ -6,7 +6,9 @@ use ecow::{eco_format, EcoString}; use unicode_math_class::MathClass; use crate::set::SyntaxSet; -use crate::{ast, is_ident, is_newline, set, LexMode, Lexer, SyntaxKind, SyntaxNode}; +use crate::{ + ast, is_ident, is_newline, set, LexMode, Lexer, SyntaxError, SyntaxKind, SyntaxNode, +}; /// Parses a source file. pub fn parse(text: &str) -> SyntaxNode { @@ -1760,8 +1762,8 @@ impl<'s> Parser<'s> { fn save(&mut self) { let text = self.current_text(); if self.at(SyntaxKind::Error) { - let message = self.lexer.take_error().unwrap(); - self.nodes.push(SyntaxNode::error(message, text)); + let error = self.lexer.take_error().unwrap(); + self.nodes.push(SyntaxNode::error(error, text)); } else { self.nodes.push(SyntaxNode::leaf(self.current, text)); } @@ -1838,7 +1840,8 @@ impl<'s> Parser<'s> { /// Produce an error that the given `thing` was expected at the position /// of the marker `m`. fn expected_at(&mut self, m: Marker, thing: &str) { - let error = SyntaxNode::error(eco_format!("expected {thing}"), ""); + let error = + SyntaxNode::error(SyntaxError::new(eco_format!("expected {thing}")), ""); self.nodes.insert(m.0, error); } diff --git a/tests/suite/foundations/array.typ b/tests/suite/foundations/array.typ index 1df83fc61..6228f471b 100644 --- a/tests/suite/foundations/array.typ +++ b/tests/suite/foundations/array.typ @@ -22,6 +22,7 @@ --- array-bad-token --- // Error: 4-6 unexpected end of block comment +// Hint: 4-6 consider escaping the `*` with a backslash or opening the block comment with `/*` #(1*/2) --- array-bad-number-suffix --- diff --git a/tests/suite/scripting/call.typ b/tests/suite/scripting/call.typ index e79fc949e..34344608a 100644 --- a/tests/suite/scripting/call.typ +++ b/tests/suite/scripting/call.typ @@ -80,6 +80,7 @@ --- call-args-bad-token --- // Error: 10-12 unexpected end of block comment +// Hint: 10-12 consider escaping the `*` with a backslash or opening the block comment with `/*` #func(a:1*/) --- call-args-missing-comma --- diff --git a/tests/suite/syntax/comment.typ b/tests/suite/syntax/comment.typ index ac3e1943f..5e7e0ee69 100644 --- a/tests/suite/syntax/comment.typ +++ b/tests/suite/syntax/comment.typ @@ -37,6 +37,7 @@ Second part --- comment-block-unclosed --- // End should not appear without start. // Error: 7-9 unexpected end of block comment +// Hint: 7-9 consider escaping the `*` with a backslash or opening the block comment with `/*` /* */ */ // Unterminated is okay.