diff --git a/crates/typst-syntax/src/lexer.rs b/crates/typst-syntax/src/lexer.rs index aacbee62e..f1d29fb3e 100644 --- a/crates/typst-syntax/src/lexer.rs +++ b/crates/typst-syntax/src/lexer.rs @@ -88,8 +88,10 @@ impl Lexer<'_> { } } -/// Shared. +/// Shared methods with all [`LexMode`]. impl Lexer<'_> { + /// Proceed to the next token and return its [`SyntaxKind`]. Note the + /// token could be a [trivia](SyntaxKind::is_trivia). pub fn next(&mut self) -> SyntaxKind { if self.mode == LexMode::Raw { let Some((kind, end)) = self.raw.pop() else { @@ -121,6 +123,7 @@ impl Lexer<'_> { } } + /// Eat whitespace characters greedily. fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind { let more = self.s.eat_while(|c| is_space(c, self.mode)); let newlines = match c { @@ -760,7 +763,7 @@ impl ScannerExt for Scanner<'_> { } } -/// Whether a character will become a Space token in Typst +/// Whether a character will become a [`SyntaxKind::Space`] token. #[inline] fn is_space(character: char, mode: LexMode) -> bool { match mode { diff --git a/crates/typst-syntax/src/parser.rs b/crates/typst-syntax/src/parser.rs index 50032898e..17f08153d 100644 --- a/crates/typst-syntax/src/parser.rs +++ b/crates/typst-syntax/src/parser.rs @@ -1748,15 +1748,27 @@ impl<'s> Parser<'s> { } } + fn next_non_trivia(lexer: &mut Lexer<'s>) -> SyntaxKind { + loop { + let next = lexer.next(); + // Loop is terminatable, because SyntaxKind::Eof is not a trivia. + if !next.is_trivia() { + break next; + } + } + } + fn lex(&mut self) { self.current_start = self.lexer.cursor(); self.current = self.lexer.next(); + + // Special cases to handle newlines in code mode. if self.lexer.mode() == LexMode::Code && self.lexer.newline() && match self.newline_modes.last() { Some(NewlineMode::Continue) => false, Some(NewlineMode::Contextual) => !matches!( - self.lexer.clone().next(), + Self::next_non_trivia(&mut self.lexer.clone()), SyntaxKind::Else | SyntaxKind::Dot ), Some(NewlineMode::Stop) => true, diff --git a/tests/typ/bugs/newline-mode.typ b/tests/typ/bugs/newline-mode.typ index 30545eeee..04333cc5e 100644 --- a/tests/typ/bugs/newline-mode.typ +++ b/tests/typ/bugs/newline-mode.typ @@ -22,3 +22,63 @@ else { ("1", "2") } + +--- +// Ref: false +#test({ + "hi 1" + + .clusters() +}, ("h", "i", " ", "1")) + +--- +// Ref: false +#test({ + "hi 2"// comment + .clusters() +}, ("h", "i", " ", "2")) + +--- +// Ref: false +#test({ + "hi 3"/* comment */ + .clusters() +}, ("h", "i", " ", "3")) + +--- +// Ref: false +#test({ + "hi 4" + // comment + .clusters() +}, ("h", "i", " ", "4")) + +--- +// Ref: false +#test({ + "hi 5" + /*comment*/.clusters() +}, ("h", "i", " ", "5")) + +--- +// Ref: false +#test({ + "hi 6" + // comment + + + /* comment */ + .clusters() +}, ("h", "i", " ", "6")) + +--- +// Ref: false +#test({ + let foo(x) = { + if x < 0 { "negative" } + // comment + else { "non-negative" } + } + + foo(1) +}, "non-negative")