diff --git a/Cargo.lock b/Cargo.lock index 628bbd244..8ed0897ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2059,6 +2059,7 @@ dependencies = [ "serde_json", "thiserror", "walkdir", + "yaml-rust", ] [[package]] diff --git a/NOTICE b/NOTICE index d2ebb9da7..93a412b45 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,27 @@ Licenses for third party components used by this project can be found below. +================================================================================ +The 0BSD License applies to: + +* The S-Expression sublime-syntax in `assets/files/SExpressions.sublime-syntax` + which is adapted from the S-Expression syntax definition in the Sublime Text + package `S-Expressions` (https://github.com/whitequark/Sublime-S-Expressions) + +BSD Zero Clause License (0BSD) + +Permission to use, copy, modify, and/or distribute this software for +any purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN +AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +================================================================================ + ================================================================================ The MIT License applies to: diff --git a/assets/files/SExpressions.sublime-syntax b/assets/files/SExpressions.sublime-syntax new file mode 100644 index 000000000..a18dd9133 --- /dev/null +++ b/assets/files/SExpressions.sublime-syntax @@ -0,0 +1,73 @@ +%YAML 1.2 +--- +name: S Expressions +file_extensions: ["sexp"] +scope: source.sexpr + +contexts: + main: + - match: '(;+).*$' + scope: comment.line.sexpr + captures: + 1: punctuation.definition.comment.sexpr + - match: '#;' + scope: punctuation.definition.comment.sexpr + push: comment + - match: '#\|' + scope: punctuation.definition.comment.sexpr + push: block_comment + + - match: '"' + scope: punctuation.definition.string.begin.sexpr + push: string_unquote + - match: '\d+\.\d+' + scope: constant.numeric.float.sexpr + - match: '\d+' + scope: constant.numeric.integer.sexpr + - match: '\w+' + scope: constant.other.sexpr + - match: '\(' + scope: punctuation.section.parens.begin.sexpr + push: main_rparen + - match: '\)' + scope: invalid.illegal.stray-paren-end + + string_unquote: + - meta_scope: string.quoted.double.sexpr + - match: '""' + scope: constant.character.escape.sexpr + - match: '"' + scope: punctuation.definition.string.end.sexpr + pop: true + + main_rparen: + - match: '\)' + scope: punctuation.section.parens.end.sexpr + pop: true + - include: main + + comment: + - meta_scope: comment.block.sexpr + - match: '\(' + set: comment_rparen + + comment_lparen: + - meta_scope: comment.block.sexpr + - match: '\(' + push: comment_rparen + - match: '\)' + scope: invalid.illegal.stray-paren-end + + comment_rparen: + - meta_scope: comment.block.sexpr + - match: '\)' + pop: true + - include: comment + + block_comment: + - meta_scope: comment.block.sexpr + - match: '#\|' + push: block_comment + - match: '\|#' + scope: punctuation.definition.comment.sexpr + pop: true \ No newline at end of file diff --git a/crates/typst-library/Cargo.toml b/crates/typst-library/Cargo.toml index 23c37d75e..86c3ab5aa 100644 --- a/crates/typst-library/Cargo.toml +++ b/crates/typst-library/Cargo.toml @@ -39,7 +39,7 @@ rustybuzz = "0.7" serde_json = "1" serde_yaml = "0.8" smallvec = "1.10" -syntect = { version = "5", default-features = false, features = ["parsing", "regex-fancy"] } +syntect = { version = "5", default-features = false, features = ["parsing", "regex-fancy", "yaml-load"] } time = { version = "0.3.20", features = ["formatting"] } toml = { version = "0.7.3", default-features = false, features = ["parse"] } tracing = "0.1.37" diff --git a/crates/typst-library/src/layout/par.rs b/crates/typst-library/src/layout/par.rs index 6b914e808..6b862a23f 100644 --- a/crates/typst-library/src/layout/par.rs +++ b/crates/typst-library/src/layout/par.rs @@ -119,11 +119,11 @@ pub struct ParElem { } impl Construct for ParElem { - fn construct(_: &mut Vm, args: &mut Args) -> SourceResult { + fn construct(vm: &mut Vm, args: &mut Args) -> SourceResult { // The paragraph constructor is special: It doesn't create a paragraph // element. Instead, it just ensures that the passed content lives in a // separate paragraph and styles it. - let styles = Self::set(args)?; + let styles = Self::set(vm, args)?; let body = args.expect::("body")?; Ok(Content::sequence([ ParbreakElem::new().pack(), diff --git a/crates/typst-library/src/text/mod.rs b/crates/typst-library/src/text/mod.rs index ff8cbad86..3c3ccdeff 100644 --- a/crates/typst-library/src/text/mod.rs +++ b/crates/typst-library/src/text/mod.rs @@ -520,11 +520,11 @@ impl TextElem { } impl Construct for TextElem { - fn construct(_: &mut Vm, args: &mut Args) -> SourceResult { + fn construct(vm: &mut Vm, args: &mut Args) -> SourceResult { // The text constructor is special: It doesn't create a text element. // Instead, it leaves the passed argument structurally unchanged, but // styles all text in it. - let styles = Self::set(args)?; + let styles = Self::set(vm, args)?; let body = args.expect::("body")?; Ok(body.styled_with_map(styles)) } diff --git a/crates/typst-library/src/text/raw.rs b/crates/typst-library/src/text/raw.rs index 873f106ac..4d4cb7102 100644 --- a/crates/typst-library/src/text/raw.rs +++ b/crates/typst-library/src/text/raw.rs @@ -1,6 +1,13 @@ +use std::hash::Hash; +use std::sync::Arc; + use once_cell::sync::Lazy; +use once_cell::unsync::Lazy as UnsyncLazy; use syntect::highlighting as synt; +use syntect::parsing::{SyntaxDefinition, SyntaxSet, SyntaxSetBuilder}; +use typst::diag::FileError; use typst::syntax::{self, LinkedNode}; +use typst::util::Bytes; use super::{ FontFamily, FontList, Hyphenate, LinebreakElem, SmartQuoteElem, TextElem, TextSize, @@ -126,6 +133,33 @@ pub struct RawElem { /// ```` #[default(HorizontalAlign(GenAlign::Start))] pub align: HorizontalAlign, + + /// One or multiple additional syntax definitions to load. The syntax + /// definitions should be in the `sublime-syntax` file format. + /// + /// ````example + /// #set raw(syntaxes: "SExpressions.sublime-syntax") + /// + /// ```sexp + /// (defun factorial (x) + /// (if (zerop x) + /// ; with a comment + /// 1 + /// (* x (factorial (- x 1))))) + /// ``` + /// ```` + #[parse( + let (syntaxes, data) = parse_syntaxes(vm, args)?; + syntaxes + )] + #[fold] + pub syntaxes: SyntaxPaths, + + /// The raw file buffers. + #[internal] + #[parse(data)] + #[fold] + pub data: Vec, } impl RawElem { @@ -163,6 +197,9 @@ impl Show for RawElem { .map(to_typst) .map_or(Color::BLACK, Color::from); + let extra_syntaxes = + UnsyncLazy::new(|| load(&self.syntaxes(styles), &self.data(styles)).unwrap()); + let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) { let root = match lang.as_deref() { Some("typc") => syntax::parse_code(&text), @@ -181,9 +218,16 @@ impl Show for RawElem { ); Content::sequence(seq) - } else if let Some(syntax) = - lang.and_then(|token| SYNTAXES.find_syntax_by_token(&token)) - { + } else if let Some((syntax_set, syntax)) = lang.and_then(|token| { + SYNTAXES + .find_syntax_by_token(&token) + .map(|syntax| (&*SYNTAXES, syntax)) + .or_else(|| { + extra_syntaxes + .find_syntax_by_token(&token) + .map(|syntax| (&**extra_syntaxes, syntax)) + }) + }) { let mut seq = vec![]; let mut highlighter = syntect::easy::HighlightLines::new(syntax, &THEME); for (i, line) in text.lines().enumerate() { @@ -192,7 +236,7 @@ impl Show for RawElem { } for (style, piece) in - highlighter.highlight_line(line, &SYNTAXES).into_iter().flatten() + highlighter.highlight_line(line, syntax_set).into_iter().flatten() { seq.push(styled(piece, foreground.into(), style)); } @@ -319,6 +363,71 @@ fn to_syn(RgbaColor { r, g, b, a }: RgbaColor) -> synt::Color { synt::Color { r, g, b, a } } +/// A list of bibliography file paths. +#[derive(Debug, Default, Clone, Hash)] +pub struct SyntaxPaths(Vec); + +cast! { + SyntaxPaths, + self => self.0.into_value(), + v: EcoString => Self(vec![v]), + v: Array => Self(v.into_iter().map(Value::cast).collect::>()?), +} + +impl Fold for SyntaxPaths { + type Output = Self; + + fn fold(mut self, outer: Self::Output) -> Self::Output { + self.0.extend(outer.0); + self + } +} + +/// Load a syntax set from a list of syntax file paths. +#[comemo::memoize] +fn load(paths: &SyntaxPaths, bytes: &[Bytes]) -> StrResult> { + let mut out = SyntaxSetBuilder::new(); + + // We might have multiple sublime-syntax/yaml files + for (path, bytes) in paths.0.iter().zip(bytes.iter()) { + let src = std::str::from_utf8(bytes).map_err(|_| FileError::InvalidUtf8)?; + out.add( + SyntaxDefinition::load_from_str(src, false, None) + .map_err(|e| eco_format!("failed to parse syntax file `{path}`: {e}"))?, + ); + } + + Ok(Arc::new(out.build())) +} + +/// Function to parse the syntaxes argument. +/// Much nicer than having it be part of the `element` macro. +fn parse_syntaxes( + vm: &mut Vm, + args: &mut Args, +) -> SourceResult<(Option, Option>)> { + let Some(Spanned { v: paths, span }) = + args.named::>("syntaxes")? + else { + return Ok((None, None)); + }; + + // Load syntax files. + let data = paths + .0 + .iter() + .map(|path| { + let id = vm.location().join(path).at(span)?; + vm.world().file(id).at(span) + }) + .collect::>>()?; + + // Check that parsing works. + let _ = load(&paths, &data).at(span)?; + + Ok((Some(paths), Some(data))) +} + /// The syntect syntax definitions. /// /// Code for syntax set generation is below. The `syntaxes` directory is from diff --git a/crates/typst-macros/src/element.rs b/crates/typst-macros/src/element.rs index 6ce91fcb2..86a320bac 100644 --- a/crates/typst-macros/src/element.rs +++ b/crates/typst-macros/src/element.rs @@ -523,6 +523,7 @@ fn create_set_impl(element: &Elem) -> TokenStream { quote! { impl ::typst::model::Set for #ident { fn set( + vm: &mut Vm, args: &mut ::typst::eval::Args, ) -> ::typst::diag::SourceResult<::typst::model::Styles> { let mut styles = ::typst::model::Styles::new(); diff --git a/crates/typst/src/eval/mod.rs b/crates/typst/src/eval/mod.rs index fe28e3f3b..97cad97db 100644 --- a/crates/typst/src/eval/mod.rs +++ b/crates/typst/src/eval/mod.rs @@ -1471,7 +1471,7 @@ impl Eval for ast::SetRule { }) .at(target.span())?; let args = self.args().eval(vm)?; - Ok(target.set(args)?.spanned(self.span())) + Ok(target.set(vm, args)?.spanned(self.span())) } } diff --git a/crates/typst/src/model/content.rs b/crates/typst/src/model/content.rs index 015f8b768..373b64203 100644 --- a/crates/typst/src/model/content.rs +++ b/crates/typst/src/model/content.rs @@ -7,8 +7,8 @@ use comemo::Prehashed; use ecow::{eco_format, EcoString, EcoVec}; use super::{ - element, Behave, Behaviour, ElemFunc, Element, Fold, Guard, Label, Locatable, - Location, Recipe, Selector, Style, Styles, Synthesize, + element, Behave, Behaviour, ElemFunc, Element, Guard, Label, Locatable, Location, + Recipe, Selector, Style, Styles, Synthesize, }; use crate::diag::{SourceResult, StrResult}; use crate::doc::Meta; @@ -588,15 +588,6 @@ impl Behave for MetaElem { } } -impl Fold for Vec { - type Output = Self; - - fn fold(mut self, outer: Self::Output) -> Self::Output { - self.extend(outer); - self - } -} - /// Tries to extract the plain-text representation of the element. pub trait PlainText { /// Write this element's plain text into the given buffer. diff --git a/crates/typst/src/model/element.rs b/crates/typst/src/model/element.rs index c673ee41a..27010cd0d 100644 --- a/crates/typst/src/model/element.rs +++ b/crates/typst/src/model/element.rs @@ -32,7 +32,7 @@ pub trait Construct { /// An element's set rule. pub trait Set { /// Parse relevant arguments into style properties for this element. - fn set(args: &mut Args) -> SourceResult; + fn set(vm: &mut Vm, args: &mut Args) -> SourceResult; } /// An element's function. @@ -80,8 +80,8 @@ impl ElemFunc { } /// Execute the set rule for the element and return the resulting style map. - pub fn set(self, mut args: Args) -> SourceResult { - let styles = (self.0.set)(&mut args)?; + pub fn set(self, vm: &mut Vm, mut args: Args) -> SourceResult { + let styles = (self.0.set)(vm, &mut args)?; args.finish()?; Ok(styles) } @@ -128,7 +128,7 @@ pub struct NativeElemFunc { /// The element's constructor. pub construct: fn(&mut Vm, &mut Args) -> SourceResult, /// The element's set rule. - pub set: fn(&mut Args) -> SourceResult, + pub set: fn(&mut Vm, &mut Args) -> SourceResult, /// Details about the function. pub info: Lazy, } diff --git a/crates/typst/src/model/styles.rs b/crates/typst/src/model/styles.rs index 23748a3ff..3a6a0b98a 100644 --- a/crates/typst/src/model/styles.rs +++ b/crates/typst/src/model/styles.rs @@ -748,3 +748,12 @@ where self.map(|inner| inner.fold(outer.unwrap_or_default())) } } + +impl Fold for Vec { + type Output = Vec; + + fn fold(mut self, outer: Self::Output) -> Self::Output { + self.extend(outer); + self + } +} diff --git a/tests/ref/text/raw-syntaxes.png b/tests/ref/text/raw-syntaxes.png new file mode 100644 index 000000000..ada751e09 Binary files /dev/null and b/tests/ref/text/raw-syntaxes.png differ diff --git a/tests/typ/text/raw-syntaxes.typ b/tests/typ/text/raw-syntaxes.typ new file mode 100644 index 000000000..5863e648c --- /dev/null +++ b/tests/typ/text/raw-syntaxes.typ @@ -0,0 +1,14 @@ +// Test code highlighting with custom syntaxes. + +--- +#set page(width: 180pt) +#set text(6pt) +#set raw(syntaxes: "/files/SExpressions.sublime-syntax") + +```sexp +(defun factorial (x) + (if (zerop x) + ; with a comment + 1 + (* x (factorial (- x 1))))) +```