Fix newline parsing behavior in code mode (#3780)

crates/typst-syntax/src/lexer.rs

@@ -88,8 +88,10 @@ impl Lexer<'_> {
     }
 }
 
-/// Shared.
+/// Shared methods with all [`LexMode`].
 impl Lexer<'_> {
+    /// Proceed to the next token and return its [`SyntaxKind`]. Note the
+    /// token could be a [trivia](SyntaxKind::is_trivia).
     pub fn next(&mut self) -> SyntaxKind {
         if self.mode == LexMode::Raw {
             let Some((kind, end)) = self.raw.pop() else {
@@ -121,6 +123,7 @@ impl Lexer<'_> {
         }
     }
 
+    /// Eat whitespace characters greedily.
     fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind {
         let more = self.s.eat_while(|c| is_space(c, self.mode));
         let newlines = match c {
@@ -760,7 +763,7 @@ impl ScannerExt for Scanner<'_> {
     }
 }
 
-/// Whether a character will become a Space token in Typst
+/// Whether a character will become a [`SyntaxKind::Space`] token.
 #[inline]
 fn is_space(character: char, mode: LexMode) -> bool {
     match mode {

crates/typst-syntax/src/parser.rs

@@ -1748,15 +1748,27 @@ impl<'s> Parser<'s> {
         }
     }
 
+    fn next_non_trivia(lexer: &mut Lexer<'s>) -> SyntaxKind {
+        loop {
+            let next = lexer.next();
+            // Loop is terminatable, because SyntaxKind::Eof is not a trivia.
+            if !next.is_trivia() {
+                break next;
+            }
+        }
+    }
+
     fn lex(&mut self) {
         self.current_start = self.lexer.cursor();
         self.current = self.lexer.next();
+
+        // Special cases to handle newlines in code mode.
         if self.lexer.mode() == LexMode::Code
             && self.lexer.newline()
             && match self.newline_modes.last() {
                 Some(NewlineMode::Continue) => false,
                 Some(NewlineMode::Contextual) => !matches!(
-                    self.lexer.clone().next(),
+                    Self::next_non_trivia(&mut self.lexer.clone()),
                     SyntaxKind::Else | SyntaxKind::Dot
                 ),
                 Some(NewlineMode::Stop) => true,

tests/typ/bugs/newline-mode.typ

@@ -22,3 +22,63 @@
 else {
   ("1", "2")
 }
+
+---
+// Ref: false
+#test({
+  "hi 1"
+
+    .clusters()
+}, ("h", "i", " ", "1"))
+
+---
+// Ref: false
+#test({
+  "hi 2"// comment
+    .clusters()
+}, ("h", "i", " ", "2"))
+
+---
+// Ref: false
+#test({
+  "hi 3"/* comment */
+    .clusters()
+}, ("h", "i", " ", "3"))
+
+---
+// Ref: false
+#test({
+  "hi 4"
+  // comment
+    .clusters()
+}, ("h", "i", " ", "4"))
+
+---
+// Ref: false
+#test({
+  "hi 5"
+  /*comment*/.clusters()
+}, ("h", "i", " ", "5"))
+
+---
+// Ref: false
+#test({
+  "hi 6"
+  // comment
+
+
+  /* comment */
+    .clusters()
+}, ("h", "i", " ", "6"))
+
+---
+// Ref: false
+#test({
+  let foo(x) = {
+    if x < 0 { "negative" }
+    // comment
+    else { "non-negative" }
+  }
+
+  foo(1)
+}, "non-negative")