Let the lexer respect linebreaks within inline raw (#3756)

crates/typst-syntax/src/kind.rs

@@ -32,7 +32,7 @@ pub enum SyntaxKind {
     RawLang,
     /// A raw delimiter consisting of 1 or 3+ backticks: `` ` ``.
     RawDelim,
-    /// A sequence of whitespace to ignore in a raw block: `    `.
+    /// A sequence of whitespace to ignore in a raw text: `    `.
     RawTrimmed,
     /// A hyperlink: `https://typst.org`.
     Link,

crates/typst-syntax/src/lexer.rs

@@ -275,10 +275,7 @@ impl Lexer<'_> {
         if backticks >= 3 {
             self.blocky_raw(start, end, backticks);
         } else {
-            // Single backtick needs no trimming or extra fancyness.
+            self.inline_raw(start, end, backticks);
-            self.s.jump(end - backticks);
-            self.push_raw(SyntaxKind::Text);
-            self.s.jump(end);
         }
 
         // Closing delimiter.
@@ -356,6 +353,25 @@ impl Lexer<'_> {
         self.s.jump(end);
     }
 
+    fn inline_raw(&mut self, start: usize, end: usize, backticks: usize) {
+        self.s.jump(start + backticks);
+
+        while self.s.cursor() < end - backticks {
+            if self.s.at(is_newline) {
+                self.push_raw(SyntaxKind::Text);
+                self.s.eat_newline();
+                self.push_raw(SyntaxKind::RawTrimmed);
+                continue;
+            }
+            self.s.eat();
+        }
+        self.push_raw(SyntaxKind::Text);
+
+        self.s.jump(end);
+    }
+
+    /// Push the current cursor that marks the end of a raw segment of
+    /// the given `kind`.
     fn push_raw(&mut self, kind: SyntaxKind) {
         let end = self.s.cursor();
         self.raw.push((kind, end));
@@ -821,7 +837,7 @@ pub fn link_prefix(text: &str) -> (&str, bool) {
     (s.before(), brackets.is_empty())
 }
 
-/// Split text at newlines.
+/// Split text at newlines. These newline characters are not kept.
 pub fn split_newlines(text: &str) -> Vec<&str> {
     let mut s = Scanner::new(text);
     let mut lines = Vec::new();

tests/typ/text/raw-code.typ

@@ -69,3 +69,18 @@ end
       </body>
     </html>
     ```
+
+---
+#set page(width: 180pt)
+#set text(6pt)
+#set raw(lang:"python")
+
+Inline raws, multiline e.g. `for i in range(10):
+  # Only this line is a comment.
+  print(i)` or otherwise e.g. `print(j)`, are colored properly.
+
+Inline raws, multiline e.g. `
+# Appears blocky due to linebreaks at the boundary.
+for i in range(10):
+  print(i)
+` or otherwise e.g. `print(j)`, are colored properly.