caisin/typst
1
0
Fork 0
mirror of https://github.com/typst/typst synced 2025-06-28 16:22:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

17. Replace while loop closures and Parser::end() to use SyntaxSet

Browse Source
This commit is contained in:
Ian Wrzesinski 2024-10-26 00:37:14 -04:00
parent 86ce443806
commit 9d9a1b1e33
4 changed files with 42 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
crates/typst-syntax/src/parser.rs
View File

@ -12,7 +12,7 @@ use crate::{ast, set, LexMode, Lexer, SyntaxError, SyntaxKind, SyntaxNode};
pub fn parse(text: &str) -> SyntaxNode { pub fn parse(text: &str) -> SyntaxNode {
let _scope = typst_timing::TimingScope::new("parse"); let _scope = typst_timing::TimingScope::new("parse");
let mut p = Parser::new(text, 0, LexMode::Markup); let mut p = Parser::new(text, 0, LexMode::Markup);
markup_exprs(&mut p, true, |_| false); markup_exprs(&mut p, true, syntax_set!(End));
p.finish_into(SyntaxKind::Markup) p.finish_into(SyntaxKind::Markup)
} }
@ -20,7 +20,7 @@ pub fn parse(text: &str) -> SyntaxNode {
pub fn parse_code(text: &str) -> SyntaxNode { pub fn parse_code(text: &str) -> SyntaxNode {
let _scope = typst_timing::TimingScope::new("parse code"); let _scope = typst_timing::TimingScope::new("parse code");
let mut p = Parser::new(text, 0, LexMode::Code); let mut p = Parser::new(text, 0, LexMode::Code);
code_exprs(&mut p, |_| false); code_exprs(&mut p, syntax_set!(End));
p.finish_into(SyntaxKind::Code) p.finish_into(SyntaxKind::Code)
} }
@ -28,19 +28,14 @@ pub fn parse_code(text: &str) -> SyntaxNode {
pub fn parse_math(text: &str) -> SyntaxNode { pub fn parse_math(text: &str) -> SyntaxNode {
let _scope = typst_timing::TimingScope::new("parse math"); let _scope = typst_timing::TimingScope::new("parse math");
let mut p = Parser::new(text, 0, LexMode::Math); let mut p = Parser::new(text, 0, LexMode::Math);
math_exprs(&mut p, |_| false); math_exprs(&mut p, syntax_set!(End));
p.finish_into(SyntaxKind::Math) p.finish_into(SyntaxKind::Math)
} }
/// Parses markup expressions until a stop condition is met. /// Parses markup expressions until a stop condition is met.
fn markup( fn markup(p: &mut Parser, at_start: bool, wrap_trivia: bool, stop_set: SyntaxSet) {
p: &mut Parser,
at_start: bool,
wrap_trivia: bool,
stop: impl FnMut(&Parser) -> bool,
) {
let m = if wrap_trivia { p.before_trivia() } else { p.marker() }; let m = if wrap_trivia { p.before_trivia() } else { p.marker() };
markup_exprs(p, at_start, stop); markup_exprs(p, at_start, stop_set);
if wrap_trivia { if wrap_trivia {
p.flush_trivia(); p.flush_trivia();
} }
@ -48,18 +43,15 @@ fn markup(
} }
/// Parses a sequence of markup expressions. /// Parses a sequence of markup expressions.
fn markup_exprs( fn markup_exprs(p: &mut Parser, mut at_start: bool, stop_set: SyntaxSet) {
p: &mut Parser, debug_assert!(stop_set.contains(SyntaxKind::End));
mut at_start: bool,
mut stop: impl FnMut(&Parser) -> bool,
) {
at_start |= p.had_newline(); at_start |= p.had_newline();
let mut nesting: usize = 0; let mut nesting: usize = 0;
while !p.end() { loop {
match p.current() { match p.current() {
SyntaxKind::LeftBracket => nesting += 1, SyntaxKind::LeftBracket => nesting += 1,
SyntaxKind::RightBracket if nesting > 0 => nesting -= 1, SyntaxKind::RightBracket if nesting > 0 => nesting -= 1,
_ if stop(p) => break, _ if p.at_set(stop_set) => break,
_ => {} _ => {}
} }
markup_expr(p, at_start); markup_expr(p, at_start);
@ -73,15 +65,16 @@ pub(super) fn reparse_markup(
range: Range<usize>, range: Range<usize>,
at_start: &mut bool, at_start: &mut bool,
nesting: &mut usize, nesting: &mut usize,
mut stop: impl FnMut(SyntaxKind) -> bool, top_level: bool,
) -> Option<Vec<SyntaxNode>> { ) -> Option<Vec<SyntaxNode>> {
let mut p = Parser::new(text, range.start, LexMode::Markup); let mut p = Parser::new(text, range.start, LexMode::Markup);
*at_start |= p.had_newline(); *at_start |= p.had_newline();
while !p.end() && p.current_start() < range.end { while p.current_start() < range.end {
match p.current() { match p.current() {
SyntaxKind::LeftBracket => *nesting += 1, SyntaxKind::LeftBracket => *nesting += 1,
SyntaxKind::RightBracket if *nesting > 0 => *nesting -= 1, SyntaxKind::RightBracket if *nesting > 0 => *nesting -= 1,
_ if stop(p.current()) => break, SyntaxKind::RightBracket if !top_level => break,
SyntaxKind::End => break,
_ => {} _ => {}
} }
markup_expr(&mut p, *at_start); markup_expr(&mut p, *at_start);
@ -132,7 +125,7 @@ fn strong(p: &mut Parser) {
p.with_nl_mode(AtNewline::StopParBreak, |p| { p.with_nl_mode(AtNewline::StopParBreak, |p| {
let m = p.marker(); let m = p.marker();
p.assert(SyntaxKind::Star); p.assert(SyntaxKind::Star);
markup(p, false, true, |p| p.at_set(syntax_set!(Star, RightBracket))); markup(p, false, true, syntax_set!(Star, RightBracket, End));
p.expect_closing_delimiter(m, SyntaxKind::Star); p.expect_closing_delimiter(m, SyntaxKind::Star);
p.wrap(m, SyntaxKind::Strong); p.wrap(m, SyntaxKind::Strong);
}); });
@ -143,7 +136,7 @@ fn emph(p: &mut Parser) {
p.with_nl_mode(AtNewline::StopParBreak, |p| { p.with_nl_mode(AtNewline::StopParBreak, |p| {
let m = p.marker(); let m = p.marker();
p.assert(SyntaxKind::Underscore); p.assert(SyntaxKind::Underscore);
markup(p, false, true, |p| p.at_set(syntax_set!(Underscore, RightBracket))); markup(p, false, true, syntax_set!(Underscore, RightBracket, End));
p.expect_closing_delimiter(m, SyntaxKind::Underscore); p.expect_closing_delimiter(m, SyntaxKind::Underscore);
p.wrap(m, SyntaxKind::Emph); p.wrap(m, SyntaxKind::Emph);
}); });
@ -154,7 +147,7 @@ fn heading(p: &mut Parser) {
p.with_nl_mode(AtNewline::Stop, |p| { p.with_nl_mode(AtNewline::Stop, |p| {
let m = p.marker(); let m = p.marker();
p.assert(SyntaxKind::HeadingMarker); p.assert(SyntaxKind::HeadingMarker);
markup(p, false, false, |p| p.at_set(syntax_set!(Label, RightBracket))); markup(p, false, false, syntax_set!(Label, RightBracket, End));
p.wrap(m, SyntaxKind::Heading); p.wrap(m, SyntaxKind::Heading);
}); });
} }
@ -164,7 +157,7 @@ fn list_item(p: &mut Parser) {
p.with_nl_mode(AtNewline::RequireColumn(p.current_column()), |p| { p.with_nl_mode(AtNewline::RequireColumn(p.current_column()), |p| {
let m = p.marker(); let m = p.marker();
p.assert(SyntaxKind::ListMarker); p.assert(SyntaxKind::ListMarker);
markup(p, false, false, |p| p.at_set(syntax_set!(RightBracket))); markup(p, false, false, syntax_set!(RightBracket, End));
p.wrap(m, SyntaxKind::ListItem); p.wrap(m, SyntaxKind::ListItem);
}); });
} }
@ -174,7 +167,7 @@ fn enum_item(p: &mut Parser) {
p.with_nl_mode(AtNewline::RequireColumn(p.current_column()), |p| { p.with_nl_mode(AtNewline::RequireColumn(p.current_column()), |p| {
let m = p.marker(); let m = p.marker();
p.assert(SyntaxKind::EnumMarker); p.assert(SyntaxKind::EnumMarker);
markup(p, false, false, |p| p.at(SyntaxKind::RightBracket)); markup(p, false, false, syntax_set!(RightBracket, End));
p.wrap(m, SyntaxKind::EnumItem); p.wrap(m, SyntaxKind::EnumItem);
}); });
} }
@ -185,10 +178,10 @@ fn term_item(p: &mut Parser) {
let m = p.marker(); let m = p.marker();
p.with_nl_mode(AtNewline::Stop, |p| { p.with_nl_mode(AtNewline::Stop, |p| {
p.assert(SyntaxKind::TermMarker); p.assert(SyntaxKind::TermMarker);
markup(p, false, false, |p| p.at_set(syntax_set!(Colon, RightBracket))); markup(p, false, false, syntax_set!(Colon, RightBracket, End));
}); });
p.expect(SyntaxKind::Colon); p.expect(SyntaxKind::Colon);
markup(p, false, false, |p| p.at(SyntaxKind::RightBracket)); markup(p, false, false, syntax_set!(RightBracket, End));
p.wrap(m, SyntaxKind::TermItem); p.wrap(m, SyntaxKind::TermItem);
}); });
} }
@ -208,22 +201,23 @@ fn equation(p: &mut Parser) {
let m = p.marker(); let m = p.marker();
p.enter_modes(LexMode::Math, AtNewline::Continue, |p| { p.enter_modes(LexMode::Math, AtNewline::Continue, |p| {
p.assert(SyntaxKind::Dollar); p.assert(SyntaxKind::Dollar);
math(p, |p| p.at(SyntaxKind::Dollar)); math(p, syntax_set!(Dollar, RightBracket, End));
p.expect_closing_delimiter(m, SyntaxKind::Dollar); p.expect_closing_delimiter(m, SyntaxKind::Dollar);
}); });
p.wrap(m, SyntaxKind::Equation); p.wrap(m, SyntaxKind::Equation);
} }
/// Parses the contents of a mathematical equation: `x^2 + 1`. /// Parses the contents of a mathematical equation: `x^2 + 1`.
fn math(p: &mut Parser, stop: impl FnMut(&Parser) -> bool) { fn math(p: &mut Parser, stop_set: SyntaxSet) {
let m = p.marker(); let m = p.marker();
math_exprs(p, stop); math_exprs(p, stop_set);
p.wrap(m, SyntaxKind::Math); p.wrap(m, SyntaxKind::Math);
} }
/// Parses a sequence of math expressions. /// Parses a sequence of math expressions.
fn math_exprs(p: &mut Parser, mut stop: impl FnMut(&Parser) -> bool) { fn math_exprs(p: &mut Parser, stop_set: SyntaxSet) {
while !p.end() && !stop(p) { debug_assert!(stop_set.contains(SyntaxKind::End));
while !p.at_set(stop_set) {
if p.at_set(set::MATH_EXPR) { if p.at_set(set::MATH_EXPR) {
math_expr(p); math_expr(p);
} else { } else {
@ -398,7 +392,7 @@ fn math_delimited(p: &mut Parser) {
let m = p.marker(); let m = p.marker();
p.eat(); p.eat();
let m2 = p.marker(); let m2 = p.marker();
while !p.end() && !p.at(SyntaxKind::Dollar) { while !p.at_set(syntax_set!(Dollar, End)) {
if math_class(p.current_text()) == Some(MathClass::Closing) { if math_class(p.current_text()) == Some(MathClass::Closing) {
p.wrap(m2, SyntaxKind::Math); p.wrap(m2, SyntaxKind::Math);
p.eat(); p.eat();
@ -466,7 +460,7 @@ fn math_args(p: &mut Parser) {
// The number of math expressions per argument. // The number of math expressions per argument.
let mut count = 0; let mut count = 0;
while !p.end() && !p.at(SyntaxKind::Dollar) { while !p.at_set(syntax_set!(Dollar, End)) {
if namable if namable
&& (p.at(SyntaxKind::MathIdent) || p.at(SyntaxKind::Text)) && (p.at(SyntaxKind::MathIdent) || p.at(SyntaxKind::Text))
&& p.text[p.current_end()..].starts_with(':') && p.text[p.current_end()..].starts_with(':')
@ -566,22 +560,23 @@ fn maybe_wrap_in_math(p: &mut Parser, arg: Marker, count: usize, named: Option<M
} }
/// Parses the contents of a code block. /// Parses the contents of a code block.
fn code(p: &mut Parser, stop: impl FnMut(&Parser) -> bool) { fn code(p: &mut Parser, stop_set: SyntaxSet) {
let m = p.marker(); let m = p.marker();
code_exprs(p, stop); code_exprs(p, stop_set);
p.wrap(m, SyntaxKind::Code); p.wrap(m, SyntaxKind::Code);
} }
/// Parses a sequence of code expressions. /// Parses a sequence of code expressions.
fn code_exprs(p: &mut Parser, mut stop: impl FnMut(&Parser) -> bool) { fn code_exprs(p: &mut Parser, stop_set: SyntaxSet) {
while !p.end() && !stop(p) { debug_assert!(stop_set.contains(SyntaxKind::End));
while !p.at_set(stop_set) {
p.with_nl_mode(AtNewline::ContextualContinue, |p| { p.with_nl_mode(AtNewline::ContextualContinue, |p| {
if !p.at_set(set::CODE_EXPR) { if !p.at_set(set::CODE_EXPR) {
p.unexpected(); p.unexpected();
return; return;
} }
code_expr(p); code_expr(p);
if !p.end() && !stop(p) && !p.eat_if(SyntaxKind::Semicolon) { if !p.at_set(stop_set) && !p.eat_if(SyntaxKind::Semicolon) {
p.expected("semicolon or line break"); p.expected("semicolon or line break");
if p.at(SyntaxKind::Label) { if p.at(SyntaxKind::Label) {
p.hint("labels can only be applied in markup mode"); p.hint("labels can only be applied in markup mode");
@ -596,7 +591,7 @@ fn code_exprs(p: &mut Parser, mut stop: impl FnMut(&Parser) -> bool) {
fn embedded_code_expr(p: &mut Parser) { fn embedded_code_expr(p: &mut Parser) {
p.enter_modes(LexMode::Code, AtNewline::Stop, |p| { p.enter_modes(LexMode::Code, AtNewline::Stop, |p| {
p.assert(SyntaxKind::Hash); p.assert(SyntaxKind::Hash);
if p.had_trivia() { if p.had_trivia() || p.end() {
p.expected("expression"); p.expected("expression");
return; return;
} }
@ -606,7 +601,7 @@ fn embedded_code_expr(p: &mut Parser) {
code_expr_prec(p, true, 0); code_expr_prec(p, true, 0);
// Consume error for things like `#12p` or `#"abc\"`.# // Consume error for things like `#12p` or `#"abc\"`.#
if !at && !p.end() { if !at {
p.unexpected(); p.unexpected();
} }
@ -777,7 +772,7 @@ fn code_block(p: &mut Parser) {
let m = p.marker(); let m = p.marker();
p.enter_modes(LexMode::Code, AtNewline::Continue, |p| { p.enter_modes(LexMode::Code, AtNewline::Continue, |p| {
p.assert(SyntaxKind::LeftBrace); p.assert(SyntaxKind::LeftBrace);
code(p, |p| p.at_set(syntax_set!(RightBrace, RightBracket, RightParen))); code(p, syntax_set!(RightBrace, RightBracket, RightParen, End));
p.expect_closing_delimiter(m, SyntaxKind::RightBrace); p.expect_closing_delimiter(m, SyntaxKind::RightBrace);
}); });
p.wrap(m, SyntaxKind::CodeBlock); p.wrap(m, SyntaxKind::CodeBlock);
@ -788,7 +783,7 @@ fn content_block(p: &mut Parser) {
let m = p.marker(); let m = p.marker();
p.enter_modes(LexMode::Markup, AtNewline::Continue, |p| { p.enter_modes(LexMode::Markup, AtNewline::Continue, |p| {
p.assert(SyntaxKind::LeftBracket); p.assert(SyntaxKind::LeftBracket);
markup(p, true, true, |p| p.at(SyntaxKind::RightBracket)); markup(p, true, true, syntax_set!(RightBracket, End));
p.expect_closing_delimiter(m, SyntaxKind::RightBracket); p.expect_closing_delimiter(m, SyntaxKind::RightBracket);
}); });
p.wrap(m, SyntaxKind::ContentBlock); p.wrap(m, SyntaxKind::ContentBlock);

8
crates/typst-syntax/src/reparser.rs
View File

@ -157,19 +157,13 @@ fn try_reparse(
let new_range = shifted..shifted + new_len; let new_range = shifted..shifted + new_len;
let at_end = end == children.len(); let at_end = end == children.len();
// Stop parsing early if this kind is encountered.
let stop_kind = match parent_kind {
Some(_) => SyntaxKind::RightBracket,
None => SyntaxKind::End,
};
// Reparse! // Reparse!
let reparsed = reparse_markup( let reparsed = reparse_markup(
text, text,
new_range.clone(), new_range.clone(),
&mut at_start, &mut at_start,
&mut nesting, &mut nesting,
|kind| kind == stop_kind, parent_kind.is_none(),
); );
if let Some(newborns) = reparsed { if let Some(newborns) = reparsed {

BIN
tests/ref/single-right-bracket.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 118 B

3
tests/suite/scripting/blocks.typ
View File

@ -135,6 +135,9 @@
// Error: 2-3 unexpected closing brace // Error: 2-3 unexpected closing brace
#} #}
--- single-right-bracket ---
]
--- content-block-in-markup-scope --- --- content-block-in-markup-scope ---
// Content blocks also create a scope. // Content blocks also create a scope.
#[#let x = 1] #[#let x = 1]